Skip to content

Commit 1092073

Browse files
committed
Refactor: split into keyvalue and structured servers
1 parent 929a330 commit 1092073

70 files changed

Lines changed: 2059 additions & 374 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

cloudata-keyvalue/pom.xml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,8 @@
1515
<dependencies>
1616
<dependency>
1717
<groupId>com.cloudata</groupId>
18-
<artifactId>cloudata-shared</artifactId>
19-
</dependency>
20-
21-
<dependency>
22-
<groupId>com.google.code.gson</groupId>
23-
<artifactId>gson</artifactId>
24-
<version>2.2.4</version>
18+
<artifactId>cloudata-server-shared</artifactId>
19+
<version>${project.version}</version>
2520
</dependency>
2621

2722
<dependency>

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueClient.java

Lines changed: 3 additions & 203 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,16 @@
44
import java.io.EOFException;
55
import java.io.IOException;
66
import java.io.InputStream;
7-
import java.io.InputStreamReader;
8-
import java.util.Iterator;
9-
import java.util.NoSuchElementException;
107

118
import javax.ws.rs.core.MediaType;
129

1310
import org.slf4j.Logger;
1411
import org.slf4j.LoggerFactory;
1512

13+
import com.cloudata.clients.StreamingRecordsetBase;
14+
import com.cloudata.util.ByteStringMessageBodyWriter;
1615
import com.cloudata.util.Hex;
17-
import com.google.common.base.Throwables;
1816
import com.google.common.io.ByteStreams;
19-
import com.google.gson.JsonElement;
20-
import com.google.gson.JsonObject;
21-
import com.google.gson.JsonParser;
2217
import com.google.protobuf.ByteString;
2318
import com.sun.jersey.api.client.Client;
2419
import com.sun.jersey.api.client.ClientResponse;
@@ -40,7 +35,6 @@ public KeyValueClient(String url) {
4035
private static Client buildClient() {
4136
ClientConfig config = new DefaultClientConfig();
4237
config.getClasses().add(ByteStringMessageBodyWriter.class);
43-
config.getClasses().add(GsonObjectMessageBodyHandler.class);
4438
Client client = Client.create(config);
4539
client.setFollowRedirects(true);
4640
return client;
@@ -65,25 +59,6 @@ public void put(long storeId, ByteString key, ByteString value) throws Exception
6559
}
6660
}
6761

68-
public void put(long storeId, ByteString key, JsonObject value) throws Exception {
69-
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key))
70-
.entity(value, MediaType.APPLICATION_JSON).post(ClientResponse.class);
71-
72-
try {
73-
int status = response.getStatus();
74-
75-
switch (status) {
76-
case 200:
77-
break;
78-
79-
default:
80-
throw new IllegalStateException("Unexpected status: " + status);
81-
}
82-
} finally {
83-
response.close();
84-
}
85-
}
86-
8762
public KeyValueEntry increment(long storeId, ByteString key) throws Exception {
8863
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key)).queryParam("action", "increment")
8964
.post(ClientResponse.class);
@@ -132,30 +107,6 @@ public String toString() {
132107

133108
}
134109

135-
public static class KeyValueJsonEntry {
136-
private final ByteString key;
137-
private final JsonElement value;
138-
139-
public KeyValueJsonEntry(ByteString key, JsonElement value) {
140-
this.key = key;
141-
this.value = value;
142-
}
143-
144-
public ByteString getKey() {
145-
return key;
146-
}
147-
148-
public JsonElement getValue() {
149-
return value;
150-
}
151-
152-
@Override
153-
public String toString() {
154-
return "JsonElement [key=" + Hex.forDebug(key) + ", value=" + value + "]";
155-
}
156-
157-
}
158-
159110
public KeyValueEntry read(long storeId, ByteString key) throws IOException {
160111
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key)).get(ClientResponse.class);
161112

@@ -182,33 +133,6 @@ public KeyValueEntry read(long storeId, ByteString key) throws IOException {
182133
}
183134
}
184135

185-
public KeyValueJsonEntry readJson(long storeId, ByteString key) throws IOException {
186-
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId, key))
187-
.accept(MediaType.APPLICATION_JSON_TYPE).get(ClientResponse.class);
188-
189-
try {
190-
int status = response.getStatus();
191-
192-
switch (status) {
193-
case 200:
194-
break;
195-
196-
case 404:
197-
return null;
198-
199-
default:
200-
throw new IllegalStateException("Unexpected status: " + status);
201-
}
202-
203-
InputStream is = response.getEntityInputStream();
204-
JsonElement value = new JsonParser().parse(new InputStreamReader(is));
205-
206-
return new KeyValueJsonEntry(key, value);
207-
} finally {
208-
response.close();
209-
}
210-
}
211-
212136
public KeyValueRecordset query(long storeId) throws IOException {
213137
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId)).get(ClientResponse.class);
214138

@@ -233,107 +157,7 @@ public KeyValueRecordset query(long storeId) throws IOException {
233157
}
234158
}
235159

236-
public KeyValueJsonRecordset queryJson(long storeId) throws IOException {
237-
ClientResponse response = CLIENT.resource(url).path(toUrlPath(storeId)).accept(MediaType.APPLICATION_JSON_TYPE)
238-
.get(ClientResponse.class);
239-
240-
try {
241-
int status = response.getStatus();
242-
243-
switch (status) {
244-
case 200:
245-
break;
246-
247-
default:
248-
throw new IllegalStateException("Unexpected status: " + status);
249-
}
250-
251-
KeyValueJsonRecordset records = new KeyValueJsonRecordset(response);
252-
response = null;
253-
return records;
254-
} finally {
255-
if (response != null) {
256-
response.close();
257-
}
258-
}
259-
}
260-
261-
static abstract class KeyValueRecordsetBase<V> implements AutoCloseable, Iterable<V> {
262-
final ClientResponse response;
263-
264-
boolean read;
265-
266-
public KeyValueRecordsetBase(ClientResponse response) {
267-
this.response = response;
268-
}
269-
270-
@Override
271-
public void close() {
272-
response.close();
273-
}
274-
275-
@Override
276-
public Iterator<V> iterator() {
277-
if (read) {
278-
throw new IllegalStateException();
279-
}
280-
read = true;
281-
InputStream is = response.getEntityInputStream();
282-
283-
final DataInputStream dis = new DataInputStream(is);
284-
285-
return new Iterator<V>() {
286-
V next;
287-
boolean done;
288-
289-
@Override
290-
public void remove() {
291-
throw new UnsupportedOperationException();
292-
}
293-
294-
@Override
295-
public V next() {
296-
ensureHaveNext();
297-
298-
if (next != null) {
299-
V ret = next;
300-
next = null;
301-
return ret;
302-
} else {
303-
throw new NoSuchElementException();
304-
}
305-
}
306-
307-
@Override
308-
public boolean hasNext() {
309-
ensureHaveNext();
310-
311-
return next != null;
312-
}
313-
314-
private void ensureHaveNext() {
315-
if (next == null) {
316-
if (!done) {
317-
try {
318-
next = read(dis);
319-
} catch (IOException e) {
320-
throw Throwables.propagate(e);
321-
}
322-
if (next == null) {
323-
done = true;
324-
}
325-
}
326-
}
327-
}
328-
329-
};
330-
331-
}
332-
333-
protected abstract V read(DataInputStream dis) throws IOException;
334-
}
335-
336-
static class KeyValueRecordset extends KeyValueRecordsetBase<KeyValueEntry> {
160+
static class KeyValueRecordset extends StreamingRecordsetBase<KeyValueEntry> {
337161
public KeyValueRecordset(ClientResponse response) {
338162
super(response);
339163
}
@@ -360,30 +184,6 @@ protected KeyValueEntry read(DataInputStream dis) throws IOException {
360184
}
361185
}
362186

363-
static class KeyValueJsonRecordset extends KeyValueRecordsetBase<KeyValueJsonEntry> {
364-
public KeyValueJsonRecordset(ClientResponse response) {
365-
super(response);
366-
}
367-
368-
@Override
369-
protected KeyValueJsonEntry read(DataInputStream dis) throws IOException {
370-
int keyLength = dis.readInt();
371-
if (keyLength == -1) {
372-
return null;
373-
}
374-
375-
ByteString key = ByteString.readFrom(ByteStreams.limit(dis, keyLength));
376-
if (key.size() != keyLength) {
377-
throw new EOFException();
378-
}
379-
380-
int valueLength = dis.readInt();
381-
JsonElement object = new JsonParser().parse(new InputStreamReader(ByteStreams.limit(dis, valueLength)));
382-
383-
return new KeyValueJsonEntry(key, object);
384-
}
385-
}
386-
387187
private String toUrlPath(long storeId, ByteString key) {
388188
return Long.toString(storeId) + "/" + Hex.toHex(key);
389189
}

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueStateMachine.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,15 @@
1515
import org.slf4j.Logger;
1616
import org.slf4j.LoggerFactory;
1717

18+
import com.cloudata.btree.BtreeQuery;
19+
import com.cloudata.btree.Keyspace;
1820
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
19-
import com.cloudata.keyvalue.btree.operation.AppendOperation;
20-
import com.cloudata.keyvalue.btree.operation.DeleteOperation;
21-
import com.cloudata.keyvalue.btree.operation.IncrementOperation;
22-
import com.cloudata.keyvalue.btree.operation.KeyOperation;
23-
import com.cloudata.keyvalue.btree.operation.Keyspace;
24-
import com.cloudata.keyvalue.btree.operation.SetOperation;
25-
import com.cloudata.keyvalue.btree.operation.Value;
26-
import com.cloudata.keyvalue.web.KeyValueQuery;
21+
import com.cloudata.keyvalue.operation.AppendOperation;
22+
import com.cloudata.keyvalue.operation.DeleteOperation;
23+
import com.cloudata.keyvalue.operation.IncrementOperation;
24+
import com.cloudata.keyvalue.operation.KeyOperation;
25+
import com.cloudata.keyvalue.operation.SetOperation;
26+
import com.cloudata.values.Value;
2727
import com.google.common.base.Throwables;
2828
import com.google.common.cache.CacheBuilder;
2929
import com.google.common.cache.CacheLoader;
@@ -165,7 +165,7 @@ public Value get(long storeId, Keyspace keyspace, ByteString key) {
165165
return keyValueStore.get(keyspace.mapToKey(key).asReadOnlyByteBuffer());
166166
}
167167

168-
public KeyValueQuery scan(long storeId) {
168+
public BtreeQuery scan(long storeId) {
169169
KeyValueStore keyValueStore = getKeyValueStore(storeId);
170170
return keyValueStore.buildQuery();
171171
}

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/KeyValueStore.java

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@
77
import org.slf4j.Logger;
88
import org.slf4j.LoggerFactory;
99

10-
import com.cloudata.keyvalue.btree.Btree;
11-
import com.cloudata.keyvalue.btree.EntryListener;
12-
import com.cloudata.keyvalue.btree.MmapPageStore;
13-
import com.cloudata.keyvalue.btree.PageStore;
14-
import com.cloudata.keyvalue.btree.ReadOnlyTransaction;
15-
import com.cloudata.keyvalue.btree.WriteTransaction;
16-
import com.cloudata.keyvalue.btree.operation.KeyOperation;
17-
import com.cloudata.keyvalue.btree.operation.Value;
18-
import com.cloudata.keyvalue.web.KeyValueQuery;
10+
import com.cloudata.btree.Btree;
11+
import com.cloudata.btree.BtreeQuery;
12+
import com.cloudata.btree.EntryListener;
13+
import com.cloudata.btree.MmapPageStore;
14+
import com.cloudata.btree.PageStore;
15+
import com.cloudata.btree.ReadOnlyTransaction;
16+
import com.cloudata.btree.WriteTransaction;
17+
import com.cloudata.keyvalue.operation.KeyOperation;
18+
import com.cloudata.values.Value;
1919

2020
public class KeyValueStore {
2121

@@ -76,8 +76,8 @@ public Value get(final ByteBuffer key) {
7676
}
7777
}
7878

79-
public KeyValueQuery buildQuery() {
80-
return new KeyValueQuery(btree);
79+
public BtreeQuery buildQuery() {
80+
return new BtreeQuery(btree);
8181
}
8282

8383
}

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/btree/operation/AppendOperation.java renamed to cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/operation/AppendOperation.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
package com.cloudata.keyvalue.btree.operation;
1+
package com.cloudata.keyvalue.operation;
22

33
import com.cloudata.keyvalue.KeyValueProto.KvAction;
44
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
5+
import com.cloudata.values.Value;
56
import com.google.protobuf.ByteString;
67

78
public class AppendOperation extends KeyOperation<Integer> {

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/btree/operation/DeleteOperation.java renamed to cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/operation/DeleteOperation.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
package com.cloudata.keyvalue.btree.operation;
1+
package com.cloudata.keyvalue.operation;
22

33
import com.cloudata.keyvalue.KeyValueProto.KvAction;
44
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
5+
import com.cloudata.values.Value;
56

67
public class DeleteOperation extends KeyOperation<Integer> {
78

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/btree/operation/IncrementOperation.java renamed to cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/operation/IncrementOperation.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
package com.cloudata.keyvalue.btree.operation;
1+
package com.cloudata.keyvalue.operation;
22

33
import org.slf4j.Logger;
44
import org.slf4j.LoggerFactory;
55

66
import com.cloudata.keyvalue.KeyValueProto.KvAction;
77
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
8+
import com.cloudata.values.Value;
89

910
public class IncrementOperation extends KeyOperation<Long> {
1011

cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/btree/operation/KeyOperation.java renamed to cloudata-keyvalue/src/main/java/com/cloudata/keyvalue/operation/KeyOperation.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
package com.cloudata.keyvalue.btree.operation;
1+
package com.cloudata.keyvalue.operation;
22

3+
import com.cloudata.btree.BtreeOperation;
34
import com.cloudata.keyvalue.KeyValueProto.KvEntry;
5+
import com.cloudata.values.Value;
46

5-
public abstract class KeyOperation<V> {
7+
public abstract class KeyOperation<V> implements BtreeOperation<V> {
68
public abstract Value doAction(Value oldValue);
79

810
public abstract KvEntry.Builder serialize();

0 commit comments

Comments
 (0)