Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Fix RowKey serialization + minor improvemetns

  • Loading branch information...
commit 163d773fa1a3b8281eb1b829ebceb08e7d96509e 1 parent 0899de0
Daniel Gómez Ferro authored
8 README.md
View
@@ -1,7 +1,7 @@
Omid
=====
-The Omid project provides transactional support for key-value stores using Snapshot Isolation. Omid stands for Optimistically transactional Management in Datastores. At this stage of the project, HBase is the only supported data-store.
+The Omid project provides transactional support for key-value stores using Snapshot Isolation. Omid stands for Optimistically transactional Management in Datasources. At this stage of the project, HBase is the only supported data-store.
If you have any question, please take a look to the [Wiki](https://github.com/yahoo/omid/wiki) or contact us at omid-project@googlegroups.com or read [the online archives](https://groups.google.com/forum/?fromgroups=#!forum/omid-project)
@@ -22,7 +22,7 @@ The core architecture of the software is described in more detail in the [Techni
Compilation
-----------
-Omid uses Maven as its build system. We are using a temporary repository for Zookeeper and Bookkeeper packages to ease the installation procedure.
+Omid uses Maven for its build system. We are using a temporary repository for Zookeeper and Bookkeeper packages to ease the installation procedure.
To compile Omid:
@@ -48,7 +48,7 @@ Hence, the order of starting should be:
3. TSO
4. Hbase
-### Zookeeper & Bookkeeper
+### Zookeeper & Bookkeepergit
Omid doesn't use anything special in Zookeeper or Bookkeeper, so you can use any install for these. However, if you are running this anywhere but localhost, you need to update the setting for HBase and TSO. See the HBase docs for changing the Zookeeper quorum. For TSO, you need to modify bin/omid.sh.
For simplicity we've included a utility script which starts Zookeeper and Bookkeeper. Run:
@@ -91,4 +91,4 @@ The logging preferences can be adjusted in src/main/resources/log4j.properties.
Acknowledgement
-------
-This project has been partially supported by the EU Comission through the Cumulo Nimbo project (FP7-257993).
+This project has been partially supported by the EU Comission through the Cumulo Nimbo project (FP7-257993).
14 conf/log4j.properties
View
@@ -1,3 +1,6 @@
+
+
+
########################################################################
#
# Copyright (c) 2011 Yahoo! Inc. All rights reserved.
@@ -39,7 +42,12 @@ log4j.appender.R.MaxBackupIndex=5
log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.O.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
-log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+#log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+#log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+
+log4j.appender.R.layout.ConversionPattern=[%d{HH:mm:ss,SSS}]%5p%6.6r[%t]%x - %C{1}.%M(%F:%L) - %m%n
+log4j.appender.O.layout.ConversionPattern=[%d{HH:mm:ss,SSS}]%5p%6.6r[%t]%x - %C{1}.%M(%F:%L) - %m%n
-log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=TRACE
+log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=INFO
+log4j.logger.com.yahoo.omid.notifications=TRACE
+log4j.logger.com.yahoo.omid.examples.notifications=TRACE
4 pom.xml
View
@@ -198,9 +198,7 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <version>0.92.0</version>
- <type>jar</type>
- <scope>compile</scope>
+ <version>0.94.3</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
13 src/main/java/com/yahoo/omid/client/TransactionalTable.java
View
@@ -28,6 +28,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.ClientScanner;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
@@ -187,8 +188,9 @@ public ResultScanner getScanner(TransactionState transactionState, Scan scan) th
Scan tsscan = new Scan(scan);
tsscan.setMaxVersions((int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
tsscan.setTimeRange(0, transactionState.getStartTimestamp() + 1);
- ClientScanner scanner = new ClientScanner(transactionState, tsscan, (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
- scanner.initialize();
+ TransactionalClientScanner scanner = new TransactionalClientScanner(
+ transactionState, getConfiguration(), tsscan, getTableName(),
+ (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
return scanner;
}
@@ -269,12 +271,13 @@ public ResultScanner getScanner(TransactionState transactionState, Scan scan) th
return filtered;
}
- protected class ClientScanner extends HTable.ClientScanner {
+ protected class TransactionalClientScanner extends ClientScanner {
private TransactionState state;
private int maxVersions;
- ClientScanner(TransactionState state, Scan scan, int maxVersions) {
- super(scan);
+ TransactionalClientScanner(TransactionState state, Configuration conf,
+ Scan scan, byte[] table, int maxVersions) throws IOException {
+ super(conf, scan, table);
this.state = state;
this.maxVersions = maxVersions;
}
14 src/main/java/com/yahoo/omid/client/regionserver/Compacter.java
View
@@ -151,7 +151,19 @@ public boolean next(List<KeyValue> result, int limit) throws IOException {
public void close() throws IOException {
internalScanner.close();
}
-
+
+ @Override
+ public boolean next(List<KeyValue> results, String metric)
+ throws IOException {
+ return next(results);
+ }
+
+ @Override
+ public boolean next(List<KeyValue> result, int limit, String metric)
+ throws IOException {
+ return next(result, limit);
+ }
+
}
private class Handler extends SimpleChannelUpstreamHandler {
8 src/main/java/com/yahoo/omid/tso/RowKey.java
View
@@ -52,11 +52,11 @@ public String toString() {
public static RowKey readObject(ChannelBuffer aInputStream) {
int hash = aInputStream.readInt();
- short len = aInputStream.readByte();
+ int len = aInputStream.readInt();
// byte[] rowId = RowKeyBuffer.nextRowKey(len);
byte[] rowId = new byte[len];
aInputStream.readBytes(rowId, 0, len);
- len = aInputStream.readByte();
+ len = aInputStream.readInt();
// byte[] tableId = RowKeyBuffer.nextRowKey(len);
byte[] tableId = new byte[len];
aInputStream.readBytes(tableId, 0, len);
@@ -69,9 +69,9 @@ public void writeObject(DataOutputStream aOutputStream)
throws IOException {
hashCode();
aOutputStream.writeInt(hash);
- aOutputStream.writeByte(rowId.length);
+ aOutputStream.writeInt(rowId.length);
aOutputStream.write(rowId,0,rowId.length);
- aOutputStream.writeByte(tableId.length);
+ aOutputStream.writeInt(tableId.length);
aOutputStream.write(tableId,0,tableId.length);
}
2  src/main/java/com/yahoo/omid/tso/ThroughputMonitor.java
View
@@ -51,7 +51,7 @@ public void run() {
long oldQueries = TSOHandler.queries;
for (;;) {
- Thread.sleep(1000);
+ Thread.sleep(10000);
long endTime = System.currentTimeMillis();
long newCounter = TSOHandler.getTransferredBytes();
2  src/main/native/Makefile
View
@@ -53,7 +53,7 @@ LIB=$(LIBDIR)/$(TSOCOMMITHASHMAP)
all: $(LIB)
$(LIB): $(OBJECTS)
- -mkdir $(LIBDIR)
+ mkdir -p $(LIBDIR)
$(CC) $(LDFLAGS) $(OBJECTS) -o $@
$(OBJECTS): $(SOURCES)
2  src/main/resources/log4j.properties
View
@@ -17,7 +17,7 @@
#
########################################################################
-log4j.rootLogger=ERROR,console
+log4j.rootLogger=WARN,console
# Logging Threshold
log4j.threshhold=ALL
11 src/test/java/com/yahoo/omid/TestUtils.java
View
@@ -16,6 +16,8 @@
package com.yahoo.omid;
+import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.Socket;
import java.net.UnknownHostException;
@@ -71,5 +73,14 @@ public static void waitForSocketNotListening(String host, int port,
LOG.info("Host " + host + ":" + port + " is up");
}
}
+
+ public static void delete(File f) throws IOException {
+ if (f.isDirectory()) {
+ for (File c : f.listFiles())
+ delete(c);
+ }
+ if (!f.delete())
+ throw new FileNotFoundException("Failed to delete file: " + f);
+ }
}
Please sign in to comment.
Something went wrong with that request. Please try again.