Permalink
Browse files

Refactor: split into keyvalue and structured servers

  • Loading branch information...
justinsb committed Dec 17, 2013
1 parent 929a330 commit 1092073b96d5d32e1f6a3faab84dba798384265a
Showing with 2,059 additions and 374 deletions.
  1. +2 −7 cloudata-keyvalue/pom.xml
  2. +3 −203 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueClient.java
  3. +9 −9 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueStateMachine.java
  4. +11 −11 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueStore.java
  5. +2 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/{btree → }/operation/AppendOperation.java
  6. +2 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/{btree → }/operation/DeleteOperation.java
  7. +2 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/{btree → }/operation/IncrementOperation.java
  8. +4 −2 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/{btree → }/operation/KeyOperation.java
  9. +2 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/{btree → }/operation/SetOperation.java
  10. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/RedisRequest.java
  11. +3 −3 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/RedisServer.java
  12. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/RedisSession.java
  13. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/AppendCommand.java
  14. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/DelCommand.java
  15. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/ExistsCommand.java
  16. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/GetCommand.java
  17. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/IncrByCommand.java
  18. +1 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/SelectCommand.java
  19. +2 −2 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/redis/commands/SetCommand.java
  20. +8 −17 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/web/KeyValueEndpoint.java
  21. +2 −1 cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/web/WebModule.java
  22. +1 −1 cloudata-keyvalue/src/test/java/com/cloudata/keyvalue/KeyValueIntegrationTest.java
  23. +21 −0 cloudata-server-shared/pom.xml
  24. +2 −3 .../com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/BranchPage.java
  25. +1 −1 .../java/com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/Btree.java
  26. +9 −0 cloudata-server-shared/src/main/java/com/cloudata/btree/BtreeOperation.java
  27. +3 −7 .../web/KeyValueQuery.java → cloudata-server-shared/src/main/java/com/cloudata/btree/BtreeQuery.java
  28. +8 −10 ...dyWriter.java → cloudata-server-shared/src/main/java/com/cloudata/btree/BtreeQueryBodyWriter.java
  29. +1 −1 ...com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/ByteBuffers.java
  30. +2 −2 ...m/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/EntryListener.java
  31. +1 −1 ...keyvalue/btree/operation → cloudata-server-shared/src/main/java/com/cloudata/btree}/Keyspace.java
  32. +4 −5 ...va/com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/LeafPage.java
  33. +1 −1 .../com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/MasterPage.java
  34. +3 −3 ...m/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/MmapPageStore.java
  35. +2 −4 ...n/java/com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/Page.java
  36. +1 −1 .../com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/PageHeader.java
  37. +2 −2 ...a/com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/PageStore.java
  38. +1 −1 ...data/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/ReadOnlyTransaction.java
  39. +1 −1 ...com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/Transaction.java
  40. +3 −4 ...cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/TransactionPage.java
  41. +4 −5 ...loudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/btree/WriteTransaction.java
  42. +10 −10 .../cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/freemap/FreeSpaceMap.java
  43. +1 −1 ...com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/freemap/RangeTree.java
  44. +1 −1 ...cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/freemap/SpaceMapEntry.java
  45. +2 −3 .../com/cloudata/keyvalue → cloudata-server-shared/src/main/java/com/cloudata}/values/JsonValue.java
  46. +2 −3 ...a/keyvalue/btree/operation → cloudata-server-shared/src/main/java/com/cloudata/values}/Value.java
  47. +12 −0 cloudata-shared/pom.xml
  48. +85 −0 cloudata-shared/src/main/java/com/cloudata/clients/StreamingRecordsetBase.java
  49. +1 −1 ...data/keyvalue → cloudata-shared/src/main/java/com/cloudata/util}/ByteStringMessageBodyWriter.java
  50. +1 −1 ...ata/keyvalue → cloudata-shared/src/main/java/com/cloudata/util}/GsonObjectMessageBodyHandler.java
  51. +42 −0 cloudata-structured/pom.xml
  52. +27 −0 cloudata-structured/src/main/assembly/assembly.xml
  53. +27 −0 cloudata-structured/src/main/bin/structured.sh
  54. +194 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredClient.java
  55. +804 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredProto.java
  56. +105 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredServer.java
  57. +158 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredStateMachine.java
  58. +83 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredStore.java
  59. +18 −0 cloudata-structured/src/main/java/com/cloudata/structured/StructuredStoreModule.java
  60. +30 −0 cloudata-structured/src/main/java/com/cloudata/structured/operation/DeleteOperation.java
  61. +33 −0 cloudata-structured/src/main/java/com/cloudata/structured/operation/SetOperation.java
  62. +15 −0 cloudata-structured/src/main/java/com/cloudata/structured/operation/StructuredOperation.java
  63. +132 −0 cloudata-structured/src/main/java/com/cloudata/structured/web/StructuredStorageEndpoint.java
  64. +18 −0 cloudata-structured/src/main/java/com/cloudata/structured/web/WebModule.java
  65. +13 −0 cloudata-structured/src/main/proto/StructuredProto.proto
  66. +18 −0 cloudata-structured/src/main/resources/logback.xml
  67. +79 −0 cloudata-structured/src/test/java/com/cloudata/keyvalue/IntegrationTestBase.java
  68. +12 −34 ...data-keyvalue → cloudata-structured}/src/test/java/com/cloudata/keyvalue/JsonIntegrationTest.java
  69. +3 −0 cloudata-structured/update-protobuf.sh
  70. +2 −0 pom.xml
@@ -15,13 +15,8 @@
<dependencies>
<dependency>
<groupId>com.cloudata</groupId>
<artifactId>cloudata-shared</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.2.4</version>
<artifactId>cloudata-server-shared</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -4,21 +4,16 @@
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.NoSuchElementException;
import javax.ws.rs.core.MediaType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudata.clients.StreamingRecordsetBase;
import com.cloudata.util.ByteStringMessageBodyWriter;
import com.cloudata.util.Hex;
import com.google.common.base.Throwables;
import com.google.common.io.ByteStreams;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.protobuf.ByteString;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
@@ -40,7 +35,6 @@ public KeyValueClient(String url) {
private static Client buildClient() {
ClientConfig config = new DefaultClientConfig();
config.getClasses().add(ByteStringMessageBodyWriter.class);
config.getClasses().add(GsonObjectMessageBodyHandler.class);
Client client = Client.create(config);
client.setFollowRedirects(true);
return client;
@@ -65,25 +59,6 @@ public void put(long storeId, ByteString key, ByteString value) throws Exception
}
}
public void put(long storeId, ByteString key, JsonObject value) throws Exception {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key))
.entity(value, MediaType.APPLICATION_JSON).post(ClientResponse.class);
try {
int status = response.getStatus();
switch (status) {
case 200:
break;
default:
throw new IllegalStateException("Unexpected status: " + status);
}
} finally {
response.close();
}
}
public KeyValueEntry increment(long storeId, ByteString key) throws Exception {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key)).queryParam("action", "increment")
.post(ClientResponse.class);
@@ -132,30 +107,6 @@ public String toString() {
}
public static class KeyValueJsonEntry {
private final ByteString key;
private final JsonElement value;
public KeyValueJsonEntry(ByteString key, JsonElement value) {
this.key = key;
this.value = value;
}
public ByteString getKey() {
return key;
}
public JsonElement getValue() {
return value;
}
@Override
public String toString() {
return "JsonElement [key=" + Hex.forDebug(key) + ", value=" + value + "]";
}
}
public KeyValueEntry read(long storeId, ByteString key) throws IOException {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key)).get(ClientResponse.class);
@@ -182,33 +133,6 @@ public KeyValueEntry read(long storeId, ByteString key) throws IOException {
}
}
public KeyValueJsonEntry readJson(long storeId, ByteString key) throws IOException {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key))
.accept(MediaType.APPLICATION_JSON_TYPE).get(ClientResponse.class);
try {
int status = response.getStatus();
switch (status) {
case 200:
break;
case 404:
return null;
default:
throw new IllegalStateException("Unexpected status: " + status);
}
InputStream is = response.getEntityInputStream();
JsonElement value = new JsonParser().parse(new InputStreamReader(is));
return new KeyValueJsonEntry(key, value);
} finally {
response.close();
}
}
public KeyValueRecordset query(long storeId) throws IOException {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId)).get(ClientResponse.class);
@@ -233,107 +157,7 @@ public KeyValueRecordset query(long storeId) throws IOException {
}
}
public KeyValueJsonRecordset queryJson(long storeId) throws IOException {
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId)).accept(MediaType.APPLICATION_JSON_TYPE)
.get(ClientResponse.class);
try {
int status = response.getStatus();
switch (status) {
case 200:
break;
default:
throw new IllegalStateException("Unexpected status: " + status);
}
KeyValueJsonRecordset records = new KeyValueJsonRecordset(response);
response = null;
return records;
} finally {
if (response != null) {
response.close();
}
}
}
static abstract class KeyValueRecordsetBase<V> implements AutoCloseable, Iterable<V> {
final ClientResponse response;
boolean read;
public KeyValueRecordsetBase(ClientResponse response) {
this.response = response;
}
@Override
public void close() {
response.close();
}
@Override
public Iterator<V> iterator() {
if (read) {
throw new IllegalStateException();
}
read = true;
InputStream is = response.getEntityInputStream();
final DataInputStream dis = new DataInputStream(is);
return new Iterator<V>() {
V next;
boolean done;
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public V next() {
ensureHaveNext();
if (next != null) {
V ret = next;
next = null;
return ret;
} else {
throw new NoSuchElementException();
}
}
@Override
public boolean hasNext() {
ensureHaveNext();
return next != null;
}
private void ensureHaveNext() {
if (next == null) {
if (!done) {
try {
next = read(dis);
} catch (IOException e) {
throw Throwables.propagate(e);
}
if (next == null) {
done = true;
}
}
}
}
};
}
protected abstract V read(DataInputStream dis) throws IOException;
}
static class KeyValueRecordset extends KeyValueRecordsetBase<KeyValueEntry> {
static class KeyValueRecordset extends StreamingRecordsetBase<KeyValueEntry> {
public KeyValueRecordset(ClientResponse response) {
super(response);
}
@@ -360,30 +184,6 @@ protected KeyValueEntry read(DataInputStream dis) throws IOException {
}
}
static class KeyValueJsonRecordset extends KeyValueRecordsetBase<KeyValueJsonEntry> {
public KeyValueJsonRecordset(ClientResponse response) {
super(response);
}
@Override
protected KeyValueJsonEntry read(DataInputStream dis) throws IOException {
int keyLength = dis.readInt();
if (keyLength == -1) {
return null;
}
ByteString key = ByteString.readFrom(ByteStreams.limit(dis, keyLength));
if (key.size() != keyLength) {
throw new EOFException();
}
int valueLength = dis.readInt();
JsonElement object = new JsonParser().parse(new InputStreamReader(ByteStreams.limit(dis, valueLength)));
return new KeyValueJsonEntry(key, object);
}
}
private String toUrlPath(long storeId, ByteString key) {
return Long.toString(storeId) + "/" + Hex.toHex(key);
}
@@ -15,15 +15,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudata.btree.BtreeQuery;
import com.cloudata.btree.Keyspace;
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
import com.cloudata.keyvalue.btree.operation.AppendOperation;
import com.cloudata.keyvalue.btree.operation.DeleteOperation;
import com.cloudata.keyvalue.btree.operation.IncrementOperation;
import com.cloudata.keyvalue.btree.operation.KeyOperation;
import com.cloudata.keyvalue.btree.operation.Keyspace;
import com.cloudata.keyvalue.btree.operation.SetOperation;
import com.cloudata.keyvalue.btree.operation.Value;
import com.cloudata.keyvalue.web.KeyValueQuery;
import com.cloudata.keyvalue.operation.AppendOperation;
import com.cloudata.keyvalue.operation.DeleteOperation;
import com.cloudata.keyvalue.operation.IncrementOperation;
import com.cloudata.keyvalue.operation.KeyOperation;
import com.cloudata.keyvalue.operation.SetOperation;
import com.cloudata.values.Value;
import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -165,7 +165,7 @@ public Value get(long storeId, Keyspace keyspace, ByteString key) {
return keyValueStore.get(keyspace.mapToKey(key).asReadOnlyByteBuffer());
}
public KeyValueQuery scan(long storeId) {
public BtreeQuery scan(long storeId) {
KeyValueStore keyValueStore = getKeyValueStore(storeId);
return keyValueStore.buildQuery();
}
@@ -7,15 +7,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudata.keyvalue.btree.Btree;
import com.cloudata.keyvalue.btree.EntryListener;
import com.cloudata.keyvalue.btree.MmapPageStore;
import com.cloudata.keyvalue.btree.PageStore;
import com.cloudata.keyvalue.btree.ReadOnlyTransaction;
import com.cloudata.keyvalue.btree.WriteTransaction;
import com.cloudata.keyvalue.btree.operation.KeyOperation;
import com.cloudata.keyvalue.btree.operation.Value;
import com.cloudata.keyvalue.web.KeyValueQuery;
import com.cloudata.btree.Btree;
import com.cloudata.btree.BtreeQuery;
import com.cloudata.btree.EntryListener;
import com.cloudata.btree.MmapPageStore;
import com.cloudata.btree.PageStore;
import com.cloudata.btree.ReadOnlyTransaction;
import com.cloudata.btree.WriteTransaction;
import com.cloudata.keyvalue.operation.KeyOperation;
import com.cloudata.values.Value;
public class KeyValueStore {
@@ -76,8 +76,8 @@ public Value get(final ByteBuffer key) {
}
}
public KeyValueQuery buildQuery() {
return new KeyValueQuery(btree);
public BtreeQuery buildQuery() {
return new BtreeQuery(btree);
}
}
@@ -1,7 +1,8 @@
package com.cloudata.keyvalue.btree.operation;
package com.cloudata.keyvalue.operation;
import com.cloudata.keyvalue.KeyValueProto.KvAction;
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
import com.cloudata.values.Value;
import com.google.protobuf.ByteString;
public class AppendOperation extends KeyOperation<Integer> {
@@ -1,7 +1,8 @@
package com.cloudata.keyvalue.btree.operation;
package com.cloudata.keyvalue.operation;
import com.cloudata.keyvalue.KeyValueProto.KvAction;
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
import com.cloudata.values.Value;
public class DeleteOperation extends KeyOperation<Integer> {
@@ -1,10 +1,11 @@
package com.cloudata.keyvalue.btree.operation;
package com.cloudata.keyvalue.operation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudata.keyvalue.KeyValueProto.KvAction;
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
import com.cloudata.values.Value;
public class IncrementOperation extends KeyOperation<Long> {
@@ -1,8 +1,10 @@
package com.cloudata.keyvalue.btree.operation;
package com.cloudata.keyvalue.operation;
import com.cloudata.btree.BtreeOperation;
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
import com.cloudata.values.Value;
public abstract class KeyOperation<V> {
public abstract class KeyOperation<V> implements BtreeOperation<V> {
public abstract Value doAction(Value oldValue);
public abstract KvEntry.Builder serialize();
Oops, something went wrong.

0 comments on commit 1092073

Please sign in to comment.