Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Added a configurable limit for scan fetching in JDBC Client.

  • Loading branch information...
commit 22a9f96d20d01639389813c082960a2d47c10323 1 parent 2aea7d5
Sudipto Das authored
412 db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClient.java
@@ -28,17 +28,21 @@
28 28 import java.util.concurrent.ConcurrentMap;
29 29
30 30 /**
31   - * A class that wraps a JDBC compliant database to allow it to be interfaced with YCSB.
32   - * This class extends {@link DB} and implements the database interface used by YCSB client.
  31 + * A class that wraps a JDBC compliant database to allow it to be interfaced
  32 + * with YCSB. This class extends {@link DB} and implements the database
  33 + * interface used by YCSB client.
33 34 *
34   - * <br> Each client will have its own instance of this class. This client is
35   - * not thread safe.
  35 + * <br>
  36 + * Each client will have its own instance of this class. This client is not
  37 + * thread safe.
36 38 *
37   - * <br> This interface expects a schema <key> <field1> <field2> <field3> ...
38   - * All attributes are of type VARCHAR. All accesses are through the primary key. Therefore,
39   - * only one index on the primary key is needed.
  39 + * <br>
  40 + * This interface expects a schema <key> <field1> <field2> <field3> ... All
  41 + * attributes are of type VARCHAR. All accesses are through the primary key.
  42 + * Therefore, only one index on the primary key is needed.
40 43 *
41   - * <p> The following options must be passed when using this database client.
  44 + * <p>
  45 + * The following options must be passed when using this database client.
42 46 *
43 47 * <ul>
44 48 * <li><b>db.driver</b> The JDBC driver class to use.</li>
@@ -46,35 +50,32 @@
46 50 * <li><b>db.user</b> User name for the connection.</li>
47 51 * <li><b>db.passwd</b> Password for the connection.</li>
48 52 * </ul>
49   - *
  53 + *
50 54 * @author sudipto
51   - *
  55 + *
52 56 */
53 57 public class JdbcDBClient extends DB implements JdbcDBClientConstants {
54   -
  58 +
55 59 private ArrayList<Connection> conns;
56 60 private boolean initialized = false;
57 61 private Properties props;
58 62 private static final String DEFAULT_PROP = "";
59 63 private ConcurrentMap<StatementType, PreparedStatement> cachedStatements;
60   -
  64 + private int scanFetchLimit;
  65 +
61 66 /**
62 67 * The statement type for the prepared statements.
63 68 */
64 69 private static class StatementType {
65   -
  70 +
66 71 enum Type {
67   - INSERT(1),
68   - DELETE(2),
69   - READ(3),
70   - UPDATE(4),
71   - SCAN(5),
72   - ;
  72 + INSERT(1), DELETE(2), READ(3), UPDATE(4), SCAN(5), ;
73 73 int internalType;
  74 +
74 75 private Type(int type) {
75 76 internalType = type;
76 77 }
77   -
  78 +
78 79 int getHashCode() {
79 80 final int prime = 31;
80 81 int result = 1;
@@ -82,12 +83,12 @@ int getHashCode() {
82 83 return result;
83 84 }
84 85 }
85   -
  86 +
86 87 Type type;
87 88 int shardIndex;
88 89 int numFields;
89 90 String tableName;
90   -
  91 +
91 92 StatementType(Type type, String tableName, int numFields, int _shardIndex) {
92 93 this.type = type;
93 94 this.tableName = tableName;
@@ -130,138 +131,163 @@ public boolean equals(Object obj) {
130 131 }
131 132 }
132 133
133   - /**
134   - * For the given key, returns what shard contains data for this key
135   - *
136   - * @param key Data key to do operation on
137   - * @return Shard index
138   - */
139   - private int getShardIndexByKey(String key) {
140   - int ret = Math.abs(key.hashCode()) % conns.size();
141   - //System.out.println(conns.size() + ": Shard instance for "+ key + " (hash " + key.hashCode()+ " ) " + " is " + ret);
142   - return ret;
143   - }
  134 + /**
  135 + * For the given key, returns what shard contains data for this key
  136 + *
  137 + * @param key
  138 + * Data key to do operation on
  139 + * @return Shard index
  140 + */
  141 + private int getShardIndexByKey(String key) {
  142 + int ret = Math.abs(key.hashCode()) % conns.size();
  143 + // System.out.println(conns.size() + ": Shard instance for "+ key +
  144 + // " (hash " + key.hashCode()+ " ) " + " is " + ret);
  145 + return ret;
  146 + }
144 147
145   - /**
146   - * For the given key, returns Connection object that holds connection
147   - * to the shard that contains this key
148   - *
149   - * @param key Data key to get information for
150   - * @return Connection object
151   - */
152   - private Connection getShardConnectionByKey(String key) {
153   - return conns.get(getShardIndexByKey(key));
154   - }
  148 + /**
  149 + * For the given key, returns Connection object that holds connection to the
  150 + * shard that contains this key
  151 + *
  152 + * @param key
  153 + * Data key to get information for
  154 + * @return Connection object
  155 + */
  156 + private Connection getShardConnectionByKey(String key) {
  157 + return conns.get(getShardIndexByKey(key));
  158 + }
155 159
156   - private void cleanupAllConnections() throws SQLException {
157   - for(Connection conn: conns) {
158   - conn.close();
159   - }
  160 + private void cleanupAllConnections() throws SQLException {
  161 + for (Connection conn : conns) {
  162 + conn.close();
160 163 }
161   -
  164 + }
  165 +
162 166 /**
163   - * Initialize the database connection and set it up for sending requests to the database.
164   - * This must be called once per client.
165   - * @throws
  167 + * Initialize the database connection and set it up for sending requests to
  168 + * the database. This must be called once per client.
  169 + *
  170 + * @throws
166 171 */
167 172 @Override
168   - public void init() throws DBException {
169   - if (initialized) {
170   - System.err.println("Client connection already initialized.");
171   - return;
172   - }
173   - props = getProperties();
174   - String urls = props.getProperty(CONNECTION_URL, DEFAULT_PROP);
175   - String user = props.getProperty(CONNECTION_USER, DEFAULT_PROP);
176   - String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP);
177   - String driver = props.getProperty(DRIVER_CLASS);
  173 + public void init() throws DBException {
  174 + if (initialized) {
  175 + System.err.println("Client connection already initialized.");
  176 + return;
  177 + }
  178 + props = getProperties();
  179 + String urls = props.getProperty(CONNECTION_URL, DEFAULT_PROP);
  180 + String user = props.getProperty(CONNECTION_USER, DEFAULT_PROP);
  181 + String passwd = props.getProperty(CONNECTION_PASSWD, DEFAULT_PROP);
  182 + String driver = props.getProperty(DRIVER_CLASS);
  183 +
  184 + String fetchLimit = props.getProperty(SCAN_FETCH_LIMIT);
  185 + if (fetchLimit == null) fetchLimit = String.valueOf(SCAN_FETCH_LIMIT_DEFAULT);
  186 + try {
  187 + scanFetchLimit = Integer.parseInt(fetchLimit);
  188 + } catch (NumberFormatException e) {
  189 + System.err.println("Invalid value for scan fetch limit passed: " + fetchLimit);
  190 + scanFetchLimit = SCAN_FETCH_LIMIT_DEFAULT;
  191 + }
178 192
179   - try {
180   - if (driver != null) {
181   - Class.forName(driver);
182   - }
183   - int shardCount = 0;
184   - conns = new ArrayList<Connection>(3);
185   - for (String url: urls.split(",")) {
186   - System.out.println("Adding shard node URL: " + url);
187   - Connection conn = DriverManager.getConnection(url, user, passwd);
188   - // Since there is no explicit commit method in the DB interface, all
189   - // operations should auto commit.
190   - conn.setAutoCommit(true);
191   - shardCount++;
192   - conns.add(conn);
193   - }
  193 + try {
  194 + if (driver != null) {
  195 + Class.forName(driver);
  196 + }
  197 + int shardCount = 0;
  198 + conns = new ArrayList<Connection>(3);
  199 + for (String url : urls.split(",")) {
  200 + System.out.println("Adding shard node URL: " + url);
  201 + Connection conn = DriverManager.getConnection(url, user, passwd);
  202 + // Since there is no explicit commit method in the DB interface, all
  203 + // operations should auto commit.
  204 + conn.setAutoCommit(true);
  205 + shardCount++;
  206 + conns.add(conn);
  207 + }
194 208
195   - System.out.println("Using " + shardCount + " shards");
  209 + System.out.println("Using " + shardCount + " shards");
196 210
197   - cachedStatements = new ConcurrentHashMap<StatementType, PreparedStatement>();
198   - } catch (ClassNotFoundException e) {
199   - System.err.println("Error in initializing the JDBS driver: " + e);
200   - throw new DBException(e);
201   - } catch (SQLException e) {
202   - System.err.println("Error in database operation: " + e);
  211 + cachedStatements = new ConcurrentHashMap<StatementType, PreparedStatement>();
  212 + } catch (ClassNotFoundException e) {
  213 + System.err.println("Error in initializing the JDBS driver: " + e);
  214 + throw new DBException(e);
  215 + } catch (SQLException e) {
  216 + System.err.println("Error in database operation: " + e);
203 217 throw new DBException(e);
204 218 } catch (NumberFormatException e) {
205 219 System.err.println("Invalid value for fieldcount property. " + e);
206 220 throw new DBException(e);
207 221 }
208   - initialized = true;
209   - }
210   -
  222 + initialized = true;
  223 + }
  224 +
211 225 @Override
212   - public void cleanup() throws DBException {
213   - try {
  226 + public void cleanup() throws DBException {
  227 + try {
214 228 cleanupAllConnections();
215 229 } catch (SQLException e) {
216 230 System.err.println("Error in closing the connection. " + e);
217 231 throw new DBException(e);
218 232 }
219   - }
220   -
221   - private PreparedStatement createAndCacheInsertStatement(StatementType insertType, String key)
222   - throws SQLException {
223   - StringBuilder insert = new StringBuilder("INSERT INTO ");
224   - insert.append(insertType.tableName);
225   - insert.append(" VALUES(?");
  233 + }
  234 +
  235 + private PreparedStatement createAndCacheInsertStatement(
  236 + StatementType insertType, String key) throws SQLException {
  237 + StringBuilder insert = new StringBuilder("INSERT INTO ");
  238 + insert.append(insertType.tableName);
  239 + insert.append(" VALUES(?");
226 240 for (int i = 0; i < insertType.numFields; i++) {
227 241 insert.append(",?");
228 242 }
229 243 insert.append(");");
230   - PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(insert.toString());
231   - PreparedStatement stmt = cachedStatements.putIfAbsent(insertType, insertStatement);
232   - if (stmt == null) return insertStatement;
233   - else return stmt;
234   - }
235   -
236   - private PreparedStatement createAndCacheReadStatement(StatementType readType, String key)
237   - throws SQLException {
  244 + PreparedStatement insertStatement = getShardConnectionByKey(key)
  245 + .prepareStatement(insert.toString());
  246 + PreparedStatement stmt = cachedStatements.putIfAbsent(insertType,
  247 + insertStatement);
  248 + if (stmt == null)
  249 + return insertStatement;
  250 + else
  251 + return stmt;
  252 + }
  253 +
  254 + private PreparedStatement createAndCacheReadStatement(StatementType readType,
  255 + String key) throws SQLException {
238 256 StringBuilder read = new StringBuilder("SELECT * FROM ");
239 257 read.append(readType.tableName);
240 258 read.append(" WHERE ");
241 259 read.append(PRIMARY_KEY);
242 260 read.append(" = ");
243 261 read.append("?;");
244   - PreparedStatement readStatement = getShardConnectionByKey(key).prepareStatement(read.toString());
245   - PreparedStatement stmt = cachedStatements.putIfAbsent(readType, readStatement);
246   - if (stmt == null) return readStatement;
247   - else return stmt;
  262 + PreparedStatement readStatement = getShardConnectionByKey(key)
  263 + .prepareStatement(read.toString());
  264 + PreparedStatement stmt = cachedStatements.putIfAbsent(readType,
  265 + readStatement);
  266 + if (stmt == null)
  267 + return readStatement;
  268 + else
  269 + return stmt;
248 270 }
249   -
250   - private PreparedStatement createAndCacheDeleteStatement(StatementType deleteType, String key)
251   - throws SQLException {
  271 +
  272 + private PreparedStatement createAndCacheDeleteStatement(
  273 + StatementType deleteType, String key) throws SQLException {
252 274 StringBuilder delete = new StringBuilder("DELETE FROM ");
253 275 delete.append(deleteType.tableName);
254 276 delete.append(" WHERE ");
255 277 delete.append(PRIMARY_KEY);
256 278 delete.append(" = ?;");
257   - PreparedStatement deleteStatement = getShardConnectionByKey(key).prepareStatement(delete.toString());
258   - PreparedStatement stmt = cachedStatements.putIfAbsent(deleteType, deleteStatement);
259   - if (stmt == null) return deleteStatement;
260   - else return stmt;
  279 + PreparedStatement deleteStatement = getShardConnectionByKey(key)
  280 + .prepareStatement(delete.toString());
  281 + PreparedStatement stmt = cachedStatements.putIfAbsent(deleteType,
  282 + deleteStatement);
  283 + if (stmt == null)
  284 + return deleteStatement;
  285 + else
  286 + return stmt;
261 287 }
262   -
263   - private PreparedStatement createAndCacheUpdateStatement(StatementType updateType, String key)
264   - throws SQLException {
  288 +
  289 + private PreparedStatement createAndCacheUpdateStatement(
  290 + StatementType updateType, String key) throws SQLException {
265 291 StringBuilder update = new StringBuilder("UPDATE ");
266 292 update.append(updateType.tableName);
267 293 update.append(" SET ");
@@ -269,42 +295,56 @@ private PreparedStatement createAndCacheUpdateStatement(StatementType updateType
269 295 update.append(COLUMN_PREFIX);
270 296 update.append(i);
271 297 update.append("=?");
272   - if (i < updateType.numFields) update.append(", ");
  298 + if (i < updateType.numFields)
  299 + update.append(", ");
273 300 }
274 301 update.append(" WHERE ");
275 302 update.append(PRIMARY_KEY);
276 303 update.append(" = ?;");
277   - PreparedStatement insertStatement = getShardConnectionByKey(key).prepareStatement(update.toString());
278   - PreparedStatement stmt = cachedStatements.putIfAbsent(updateType, insertStatement);
279   - if (stmt == null) return insertStatement;
280   - else return stmt;
  304 + PreparedStatement insertStatement = getShardConnectionByKey(key)
  305 + .prepareStatement(update.toString());
  306 + PreparedStatement stmt = cachedStatements.putIfAbsent(updateType,
  307 + insertStatement);
  308 + if (stmt == null)
  309 + return insertStatement;
  310 + else
  311 + return stmt;
281 312 }
282   -
283   - private PreparedStatement createAndCacheScanStatement(StatementType scanType, String key)
284   - throws SQLException {
285   - StringBuilder select = new StringBuilder("SELECT * FROM ");
  313 +
  314 + private PreparedStatement createAndCacheScanStatement(StatementType scanType,
  315 + String key) throws SQLException {
  316 + StringBuilder select = new StringBuilder("SELECT * FROM ");
286 317 select.append(scanType.tableName);
287 318 select.append(" WHERE ");
288 319 select.append(PRIMARY_KEY);
289 320 select.append(" >= ");
290 321 select.append("?;");
291   - PreparedStatement scanStatement = getShardConnectionByKey(key).prepareStatement(select.toString());
292   - PreparedStatement stmt = cachedStatements.putIfAbsent(scanType, scanStatement);
293   - if (stmt == null) return scanStatement;
294   - else return stmt;
  322 + PreparedStatement scanStatement = getShardConnectionByKey(key)
  323 + .prepareStatement(select.toString());
  324 + // Set a limit on the number of rows fetched. This prevents the JDBC implementation
  325 + // to fetch the entire result set. Some implementations, such as that of postgres
  326 + // fetches the entire result set instead of streaming through the results.
  327 + scanStatement.setFetchSize(scanFetchLimit);
  328 + PreparedStatement stmt = cachedStatements.putIfAbsent(scanType,
  329 + scanStatement);
  330 + if (stmt == null)
  331 + return scanStatement;
  332 + else
  333 + return stmt;
295 334 }
296 335
297   - @Override
298   - public int read(String tableName, String key, Set<String> fields,
299   - HashMap<String, ByteIterator> result) {
300   - if (tableName == null) {
  336 + @Override
  337 + public int read(String tableName, String key, Set<String> fields,
  338 + HashMap<String, ByteIterator> result) {
  339 + if (tableName == null) {
301 340 return -1;
302 341 }
303 342 if (key == null) {
304 343 return -1;
305 344 }
306 345 try {
307   - StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, getShardIndexByKey(key));
  346 + StatementType type = new StatementType(StatementType.Type.READ,
  347 + tableName, 1, getShardIndexByKey(key));
308 348 PreparedStatement readStatement = cachedStatements.get(type);
309 349 if (readStatement == null) {
310 350 readStatement = createAndCacheReadStatement(type, key);
@@ -324,22 +364,24 @@ public int read(String tableName, String key, Set<String> fields,
324 364 resultSet.close();
325 365 return SUCCESS;
326 366 } catch (SQLException e) {
327   - System.err.println("Error in processing read of table " + tableName + ": "+e);
  367 + System.err.println("Error in processing read of table " + tableName
  368 + + ": " + e);
328 369 return -2;
329 370 }
330   - }
  371 + }
331 372
332   - @Override
333   - public int scan(String tableName, String startKey, int recordcount,
334   - Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
335   - if (tableName == null) {
  373 + @Override
  374 + public int scan(String tableName, String startKey, int recordcount,
  375 + Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
  376 + if (tableName == null) {
336 377 return -1;
337 378 }
338 379 if (startKey == null) {
339 380 return -1;
340 381 }
341 382 try {
342   - StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, getShardIndexByKey(startKey));
  383 + StatementType type = new StatementType(StatementType.Type.SCAN,
  384 + tableName, 1, getShardIndexByKey(startKey));
343 385 PreparedStatement scanStatement = cachedStatements.get(type);
344 386 if (scanStatement == null) {
345 387 scanStatement = createAndCacheScanStatement(type, startKey);
@@ -362,11 +404,12 @@ public int scan(String tableName, String startKey, int recordcount,
362 404 System.err.println("Error in processing scan of table: " + tableName + e);
363 405 return -2;
364 406 }
365   - }
  407 + }
366 408
367   - @Override
368   - public int update(String tableName, String key, HashMap<String, ByteIterator> values) {
369   - if (tableName == null) {
  409 + @Override
  410 + public int update(String tableName, String key,
  411 + HashMap<String, ByteIterator> values) {
  412 + if (tableName == null) {
370 413 return -1;
371 414 }
372 415 if (key == null) {
@@ -374,7 +417,8 @@ public int update(String tableName, String key, HashMap<String, ByteIterator> va
374 417 }
375 418 try {
376 419 int numFields = values.size();
377   - StatementType type = new StatementType(StatementType.Type.UPDATE, tableName, numFields, getShardIndexByKey(key));
  420 + StatementType type = new StatementType(StatementType.Type.UPDATE,
  421 + tableName, numFields, getShardIndexByKey(key));
378 422 PreparedStatement updateStatement = cachedStatements.get(type);
379 423 if (updateStatement == null) {
380 424 updateStatement = createAndCacheUpdateStatement(type, key);
@@ -385,29 +429,34 @@ public int update(String tableName, String key, HashMap<String, ByteIterator> va
385 429 }
386 430 updateStatement.setString(index, key);
387 431 int result = updateStatement.executeUpdate();
388   - if (result == 1) return SUCCESS;
389   - else return 1;
  432 + if (result == 1)
  433 + return SUCCESS;
  434 + else
  435 + return 1;
390 436 } catch (SQLException e) {
391   - System.err.println("Error in processing update to table: " + tableName + e);
  437 + System.err.println("Error in processing update to table: " + tableName
  438 + + e);
392 439 return -1;
393 440 }
394   - }
  441 + }
395 442
396   - @Override
397   - public int insert(String tableName, String key, HashMap<String, ByteIterator> values) {
398   - if (tableName == null) {
399   - return -1;
400   - }
401   - if (key == null) {
402   - return -1;
403   - }
404   - try {
405   - int numFields = values.size();
406   - StatementType type = new StatementType(StatementType.Type.INSERT, tableName, numFields, getShardIndexByKey(key));
407   - PreparedStatement insertStatement = cachedStatements.get(type);
408   - if (insertStatement == null) {
409   - insertStatement = createAndCacheInsertStatement(type, key);
410   - }
  443 + @Override
  444 + public int insert(String tableName, String key,
  445 + HashMap<String, ByteIterator> values) {
  446 + if (tableName == null) {
  447 + return -1;
  448 + }
  449 + if (key == null) {
  450 + return -1;
  451 + }
  452 + try {
  453 + int numFields = values.size();
  454 + StatementType type = new StatementType(StatementType.Type.INSERT,
  455 + tableName, numFields, getShardIndexByKey(key));
  456 + PreparedStatement insertStatement = cachedStatements.get(type);
  457 + if (insertStatement == null) {
  458 + insertStatement = createAndCacheInsertStatement(type, key);
  459 + }
411 460 insertStatement.setString(1, key);
412 461 int index = 2;
413 462 for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
@@ -415,35 +464,42 @@ public int insert(String tableName, String key, HashMap<String, ByteIterator> va
415 464 insertStatement.setString(index++, field);
416 465 }
417 466 int result = insertStatement.executeUpdate();
418   - if (result == 1) return SUCCESS;
419   - else return 1;
  467 + if (result == 1)
  468 + return SUCCESS;
  469 + else
  470 + return 1;
420 471 } catch (SQLException e) {
421   - System.err.println("Error in processing insert to table: " + tableName + e);
  472 + System.err.println("Error in processing insert to table: " + tableName
  473 + + e);
422 474 return -1;
423 475 }
424   - }
  476 + }
425 477
426   - @Override
427   - public int delete(String tableName, String key) {
428   - if (tableName == null) {
  478 + @Override
  479 + public int delete(String tableName, String key) {
  480 + if (tableName == null) {
429 481 return -1;
430 482 }
431 483 if (key == null) {
432 484 return -1;
433 485 }
434 486 try {
435   - StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, getShardIndexByKey(key));
  487 + StatementType type = new StatementType(StatementType.Type.DELETE,
  488 + tableName, 1, getShardIndexByKey(key));
436 489 PreparedStatement deleteStatement = cachedStatements.get(type);
437 490 if (deleteStatement == null) {
438 491 deleteStatement = createAndCacheDeleteStatement(type, key);
439 492 }
440 493 deleteStatement.setString(1, key);
441 494 int result = deleteStatement.executeUpdate();
442   - if (result == 1) return SUCCESS;
443   - else return 1;
  495 + if (result == 1)
  496 + return SUCCESS;
  497 + else
  498 + return 1;
444 499 } catch (SQLException e) {
445   - System.err.println("Error in processing delete to table: " + tableName + e);
  500 + System.err.println("Error in processing delete to table: " + tableName
  501 + + e);
446 502 return -1;
447 503 }
448   - }
  504 + }
449 505 }
9 db/jdbc/src/com/yahoo/ycsb/db/JdbcDBClientConstants.java
@@ -42,6 +42,15 @@
42 42 /** Default number of fields in a record. */
43 43 public static final String FIELD_COUNT_PROPERTY_DEFAULT="10";
44 44
  45 + /**
  46 + * The name of the property that specify the maximum number of entries
  47 + * prefetched when a scan is executed.
  48 + */
  49 + public static final String SCAN_FETCH_LIMIT = "scan.fetch.limit";
  50 +
  51 + /** Default limit for scan fetch. */
  52 + public static final int SCAN_FETCH_LIMIT_DEFAULT = 10;
  53 +
45 54 /** Representing a NULL value. */
46 55 public static final String NULL_VALUE = "NULL";
47 56

0 comments on commit 22a9f96

Please sign in to comment.
Something went wrong with that request. Please try again.