Skip to content

Commit

Permalink
[jdbc] solidified field ordering
Browse files Browse the repository at this point in the history 
Changes to account for the field ordering. Now correctly inserting and
updating field values so that they can be verified when using
dataintegrity=true.

Fixes brianfrankcooper#132
  • Loading branch information
@kruthar
kruthar committed Jan 27, 2016
1 parent b7d2249 commit 73ec3b1
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 37 deletions.
78 changes: 62 additions & 16 deletions jdbc/src/main/java/com/yahoo/ycsb/db/JdbcDBClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,27 @@ public class JdbcDBClient extends DB {
private static final String DEFAULT_PROP = "";
private ConcurrentMap<StatementType, PreparedStatement> cachedStatements;

/**
* Ordered field information for insert and update statements.
*/
private static class OrderedFieldInfo {
private String fieldKeys;
private List<String> fieldValues;

OrderedFieldInfo(String fieldKeys, List<String> fieldValues) {
this.fieldKeys = fieldKeys;
this.fieldValues = fieldValues;
}

String getFieldKeys() {
return fieldKeys;
}

List<String> getFieldValues() {
return fieldValues;
}
}

/**
* The statement type for the prepared statements.
*/
Expand All @@ -109,11 +130,13 @@ int getHashCode() {
private int shardIndex;
private int numFields;
private String tableName;
private String fieldString;

StatementType(Type type, String tableName, int numFields, int shardIndex) {
StatementType(Type type, String tableName, int numFields, String fieldString, int shardIndex) {
this.type = type;
this.tableName = tableName;
this.numFields = numFields;
this.fieldString = fieldString;
this.shardIndex = shardIndex;
}

Expand Down Expand Up @@ -155,6 +178,9 @@ public boolean equals(Object obj) {
if (type != other.type) {
return false;
}
if (!fieldString.equals(other.fieldString)) {
return false;
}
return true;
}
}
Expand Down Expand Up @@ -261,6 +287,7 @@ public void cleanup() throws DBException {
private PreparedStatement createAndCacheInsertStatement(StatementType insertType, String key) throws SQLException {
StringBuilder insert = new StringBuilder("INSERT INTO ");
insert.append(insertType.tableName);
insert.append(" (" + PRIMARY_KEY + "," + insertType.fieldString + ")");
insert.append(" VALUES(?");
for (int i = 0; i < insertType.numFields; i++) {
insert.append(",?");
Expand Down Expand Up @@ -304,14 +331,14 @@ private PreparedStatement createAndCacheDeleteStatement(StatementType deleteType
}

private PreparedStatement createAndCacheUpdateStatement(StatementType updateType, String key) throws SQLException {
String[] fieldKeys = updateType.fieldString.split(",");
StringBuilder update = new StringBuilder("UPDATE ");
update.append(updateType.tableName);
update.append(" SET ");
for (int i = 0; i < updateType.numFields; i++) {
update.append(COLUMN_PREFIX);
update.append(i);
for (int i = 0; i < fieldKeys.length; i++) {
update.append(fieldKeys[i]);
update.append("=?");
if (i < updateType.numFields - 1) {
if (i < fieldKeys.length - 1) {
update.append(", ");
}
}
Expand Down Expand Up @@ -349,7 +376,7 @@ private PreparedStatement createAndCacheScanStatement(StatementType scanType, St
@Override
public Status read(String tableName, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
try {
StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, getShardIndexByKey(key));
StatementType type = new StatementType(StatementType.Type.READ, tableName, 1, "", getShardIndexByKey(key));
PreparedStatement readStatement = cachedStatements.get(type);
if (readStatement == null) {
readStatement = createAndCacheReadStatement(type, key);
Expand Down Expand Up @@ -378,7 +405,7 @@ public Status read(String tableName, String key, Set<String> fields, HashMap<Str
public Status scan(String tableName, String startKey, int recordcount, Set<String> fields,
Vector<HashMap<String, ByteIterator>> result) {
try {
StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, getShardIndexByKey(startKey));
StatementType type = new StatementType(StatementType.Type.SCAN, tableName, 1, "", getShardIndexByKey(startKey));
PreparedStatement scanStatement = cachedStatements.get(type);
if (scanStatement == null) {
scanStatement = createAndCacheScanStatement(type, startKey);
Expand Down Expand Up @@ -408,14 +435,16 @@ public Status scan(String tableName, String startKey, int recordcount, Set<Strin
public Status update(String tableName, String key, HashMap<String, ByteIterator> values) {
try {
int numFields = values.size();
StatementType type = new StatementType(StatementType.Type.UPDATE, tableName, numFields, getShardIndexByKey(key));
OrderedFieldInfo fieldInfo = getFieldInfo(values);
StatementType type = new StatementType(StatementType.Type.UPDATE, tableName,
numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key));
PreparedStatement updateStatement = cachedStatements.get(type);
if (updateStatement == null) {
updateStatement = createAndCacheUpdateStatement(type, key);
}
int index = 1;
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
updateStatement.setString(index++, entry.getValue().toString());
for (String value: fieldInfo.getFieldValues()) {
updateStatement.setString(index++, value);
}
updateStatement.setString(index, key);
int result = updateStatement.executeUpdate();
Expand All @@ -433,16 +462,17 @@ public Status update(String tableName, String key, HashMap<String, ByteIterator>
public Status insert(String tableName, String key, HashMap<String, ByteIterator> values) {
try {
int numFields = values.size();
StatementType type = new StatementType(StatementType.Type.INSERT, tableName, numFields, getShardIndexByKey(key));
OrderedFieldInfo fieldInfo = getFieldInfo(values);
StatementType type = new StatementType(StatementType.Type.INSERT, tableName,
numFields, fieldInfo.getFieldKeys(), getShardIndexByKey(key));
PreparedStatement insertStatement = cachedStatements.get(type);
if (insertStatement == null) {
insertStatement = createAndCacheInsertStatement(type, key);
}
insertStatement.setString(1, key);
int index = 2;
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
String field = entry.getValue().toString();
insertStatement.setString(index++, field);
for (String value: fieldInfo.getFieldValues()) {
insertStatement.setString(index++, value);
}
int result = insertStatement.executeUpdate();
if (result == 1) {
Expand All @@ -458,7 +488,7 @@ public Status insert(String tableName, String key, HashMap<String, ByteIterator>
@Override
public Status delete(String tableName, String key) {
try {
StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, getShardIndexByKey(key));
StatementType type = new StatementType(StatementType.Type.DELETE, tableName, 1, "", getShardIndexByKey(key));
PreparedStatement deleteStatement = cachedStatements.get(type);
if (deleteStatement == null) {
deleteStatement = createAndCacheDeleteStatement(type, key);
Expand All @@ -474,4 +504,20 @@ public Status delete(String tableName, String key) {
return Status.ERROR;
}
}
}

private OrderedFieldInfo getFieldInfo(HashMap<String, ByteIterator> values) {
String fieldKeys = "";
List<String> fieldValues = new ArrayList();
int count = 0;
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
fieldKeys += entry.getKey();
if (count < values.size() - 1) {
fieldKeys += ",";
}
fieldValues.add(count, entry.getValue().toString());
count++;
}

return new OrderedFieldInfo(fieldKeys, fieldValues);
}
}
29 changes: 8 additions & 21 deletions jdbc/src/test/java/com/yahoo/ycsb/db/JdbcDBClientTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2015 Yahoo! Inc. All rights reserved.
* Copyright (c) 2015 - 2016 Yahoo! Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
Expand Down Expand Up @@ -30,9 +30,6 @@
import java.util.Properties;
import java.util.Vector;

/**
* Created by kruthar on 11/2/15.
*/
public class JdbcDBClientTest {
private static final String TEST_DB_DRIVER = "org.hsqldb.jdbc.JDBCDriver";
private static final String TEST_DB_URL = "jdbc:hsqldb:mem:ycsb";
Expand Down Expand Up @@ -153,7 +150,7 @@ private HashMap<String, ByteIterator> insertRow(String insertKey) {
public void insertTest() {
try {
String insertKey = "user0";
insertRow(insertKey);
HashMap<String, ByteIterator> insertMap = insertRow(insertKey);

ResultSet resultSet = jdbcConnection.prepareStatement(
String.format("SELECT * FROM %s", TABLE_NAME)
Expand All @@ -164,9 +161,7 @@ public void insertTest() {
// Check that all the columns have expected values
assertEquals(resultSet.getString(KEY_FIELD), insertKey);
for (int i = 0; i < 3; i++) {
// TODO: This will fail until the fix is made to insert and update fields in the correct order.
// TODO: Uncomment this assertEquals when the fix is made.
//assertEquals(resultSet.getString(FIELD_PREFIX + i), insertMap.get(FIELD_PREFIX + i));
assertEquals(resultSet.getString(FIELD_PREFIX + i), insertMap.get(FIELD_PREFIX + i).toString());
}
// Check that we do not have any more rows
assertFalse(resultSet.next());
Expand Down Expand Up @@ -224,9 +219,7 @@ public void updateTest() {
resultSet.next();
assertEquals(resultSet.getString(KEY_FIELD), "user1");
for (int i = 0; i < 3; i++) {
// TODO: This will fail until the fix is made to insert and update fields in the correct order.
// TODO: Uncomment this assertEquals when the fix is made.
//assertEquals(resultSet.getString(FIELD_PREFIX + i), updateMap.get(FIELD_PREFIX + i));
assertEquals(resultSet.getString(FIELD_PREFIX + i), updateMap.get(FIELD_PREFIX + i).toString());
}

// Ensure that user2 record was not changed
Expand All @@ -245,7 +238,7 @@ public void updateTest() {
@Test
public void readTest() {
String insertKey = "user0";
insertRow(insertKey);
HashMap<String, ByteIterator> insertMap = insertRow(insertKey);
HashSet<String> readFields = new HashSet<String>();
HashMap<String, ByteIterator> readResultMap = new HashMap<String, ByteIterator>();

Expand All @@ -254,9 +247,7 @@ public void readTest() {
jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap);
assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size());
for (String field: readFields) {
// TODO: This will fail until the fix is made to insert and update fields in the correct order.
// TODO: Uncomment this assertEquals when the fix is made.
//assertEquals("Assert " + field + " was read correctly", insertMap.get(field), readResultMap.get(field));
assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString());
}

readResultMap = new HashMap<String, ByteIterator>();
Expand All @@ -267,9 +258,7 @@ public void readTest() {
jdbcDBClient.read(TABLE_NAME, insertKey, readFields, readResultMap);
assertEquals("Assert that result has correct number of fields", readFields.size(), readResultMap.size());
for (String field: readFields) {
// TODO: This will fail until the fix is made to insert and update fields in the correct order.
// TODO: Uncomment this assertEquals when the fix is made.
//assertEquals("Assert " + field + " was read correctly", insertMap.get(field), readResultMap.get(field));
assertEquals("Assert " + field + " was read correctly", insertMap.get(field).toString(), readResultMap.get(field).toString());
}
}

Expand Down Expand Up @@ -325,9 +314,7 @@ public void scanTest() throws SQLException {
for (HashMap<String, ByteIterator> result: resultVector) {
assertEquals("Assert that this row has the correct number of fields", fieldSet.size(), result.size());
for (String field: fieldSet) {
// TODO: This will fail until the fix is made to insert and update fields in the correct order.
// TODO: Uncomment this assertEquals when the fix is made.
//assertEquals("Assert this field is correct in this row", keyMap.get(KEY_PREFIX + testIndex).get(field), result.get(field));
assertEquals("Assert this field is correct in this row", keyMap.get(KEY_PREFIX + testIndex).get(field).toString(), result.get(field).toString());
}
testIndex++;
}
Expand Down

0 comments on commit 73ec3b1

Please sign in to comment.