Fix RowKey serialization + minor improvemetns

YahooArchive · Feb 8, 2013 · 163d773 · 163d773
1 parent 0899de0
commit 163d773
Show file tree

Hide file tree

Showing 10 changed files with 55 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 Omid
 =====
 
-The Omid project provides transactional support for key-value stores using Snapshot Isolation. Omid stands for Optimistically transactional Management in Datastores. At this stage of the project, HBase is the only supported data-store.
+The Omid project provides transactional support for key-value stores using Snapshot Isolation. Omid stands for Optimistically transactional Management in Datasources. At this stage of the project, HBase is the only supported data-store.
 
 If you have any question, please take a look to the [Wiki](https://github.com/yahoo/omid/wiki) or contact us at omid-project@googlegroups.com or read [the online archives](https://groups.google.com/forum/?fromgroups=#!forum/omid-project)
 
@@ -22,7 +22,7 @@ The core architecture of the software is described in more detail in the [Techni
 Compilation
 -----------
 
-Omid uses Maven as its build system. We are using a temporary repository for Zookeeper and Bookkeeper packages to ease the installation procedure.
+Omid uses Maven for its build system. We are using a temporary repository for Zookeeper and Bookkeeper packages to ease the installation procedure.
 
 To compile Omid:
 
@@ -48,7 +48,7 @@ Hence, the order of starting should be:
 3. TSO
 4. Hbase
 
-### Zookeeper & Bookkeeper
+### Zookeeper & Bookkeepergit
 Omid doesn't use anything special in Zookeeper or Bookkeeper, so you can use any install for these. However, if you are running this anywhere but localhost, you need to update the setting for HBase and TSO. See the HBase docs for changing the Zookeeper quorum. For TSO, you need to modify bin/omid.sh.
 
 For simplicity we've included a utility script which starts Zookeeper and Bookkeeper. Run:
@@ -91,4 +91,4 @@ The logging preferences can be adjusted in src/main/resources/log4j.properties.
 
 Acknowledgement
 -------
-This project has been partially supported by the EU Comission through the Cumulo Nimbo project (FP7-257993).
+This project has been partially supported by the EU Comission through the Cumulo Nimbo project (FP7-257993).
diff --git a/conf/log4j.properties b/conf/log4j.properties
@@ -1,3 +1,6 @@
+
+
+
 ########################################################################
 #
 # Copyright (c) 2011 Yahoo! Inc. All rights reserved.
@@ -39,7 +42,12 @@ log4j.appender.R.MaxBackupIndex=5
 log4j.appender.R.layout=org.apache.log4j.PatternLayout
 log4j.appender.O.layout=org.apache.log4j.PatternLayout
 
-log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
-log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+#log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+#log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
+
+log4j.appender.R.layout.ConversionPattern=[%d{HH:mm:ss,SSS}]%5p%6.6r[%t]%x - %C{1}.%M(%F:%L) - %m%n
+log4j.appender.O.layout.ConversionPattern=[%d{HH:mm:ss,SSS}]%5p%6.6r[%t]%x - %C{1}.%M(%F:%L) - %m%n
 
-log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=TRACE
+log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=INFO
+log4j.logger.com.yahoo.omid.notifications=TRACE
+log4j.logger.com.yahoo.omid.examples.notifications=TRACE
diff --git a/pom.xml b/pom.xml
@@ -198,9 +198,7 @@
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase</artifactId>
-            <version>0.92.0</version>
-            <type>jar</type>
-            <scope>compile</scope>
+            <version>0.94.3</version>
         </dependency>
         <dependency>
             <groupId>org.apache.zookeeper</groupId>

diff --git a/src/main/java/com/yahoo/omid/client/TransactionalTable.java b/src/main/java/com/yahoo/omid/client/TransactionalTable.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.ClientScanner;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
@@ -187,8 +188,9 @@ public ResultScanner getScanner(TransactionState transactionState, Scan scan) th
       Scan tsscan = new Scan(scan);
       tsscan.setMaxVersions((int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
       tsscan.setTimeRange(0, transactionState.getStartTimestamp() + 1);
-      ClientScanner scanner = new ClientScanner(transactionState, tsscan, (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
-      scanner.initialize();
+      TransactionalClientScanner scanner = new TransactionalClientScanner(
+            transactionState, getConfiguration(), tsscan, getTableName(),
+            (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
       return scanner;
    }
 
@@ -269,12 +271,13 @@ private List<KeyValue> filter(TransactionState transactionState, List<KeyValue>
       return filtered;
    }
 
-   protected class ClientScanner extends HTable.ClientScanner {
+   protected class TransactionalClientScanner extends ClientScanner {
       private TransactionState state;
       private int maxVersions;
 
-      ClientScanner(TransactionState state, Scan scan, int maxVersions) {
-         super(scan);
+      TransactionalClientScanner(TransactionState state, Configuration conf,
+            Scan scan, byte[] table, int maxVersions) throws IOException {
+         super(conf, scan, table);
          this.state = state;
          this.maxVersions = maxVersions;
       }

diff --git a/src/main/java/com/yahoo/omid/client/regionserver/Compacter.java b/src/main/java/com/yahoo/omid/client/regionserver/Compacter.java
@@ -151,7 +151,19 @@ public boolean next(List<KeyValue> result, int limit) throws IOException {
       public void close() throws IOException {
          internalScanner.close();
       }
-
+
+      @Override
+      public boolean next(List<KeyValue> results, String metric)
+            throws IOException {
+         return next(results);
+      }
+
+      @Override
+      public boolean next(List<KeyValue> result, int limit, String metric)
+            throws IOException {
+         return next(result, limit);
+      }
+
    }
 
    private class Handler extends SimpleChannelUpstreamHandler {

diff --git a/src/main/java/com/yahoo/omid/tso/RowKey.java b/src/main/java/com/yahoo/omid/tso/RowKey.java
@@ -52,11 +52,11 @@ public String toString() {
 
    public static RowKey readObject(ChannelBuffer aInputStream) {
       int hash = aInputStream.readInt();
-      short len = aInputStream.readByte();
+      int len = aInputStream.readInt();
 //      byte[] rowId = RowKeyBuffer.nextRowKey(len);
       byte[] rowId = new byte[len];
       aInputStream.readBytes(rowId, 0, len);
-      len = aInputStream.readByte();
+      len = aInputStream.readInt();
 //      byte[] tableId = RowKeyBuffer.nextRowKey(len);
       byte[] tableId = new byte[len];
       aInputStream.readBytes(tableId, 0, len);
@@ -69,9 +69,9 @@ public void writeObject(DataOutputStream aOutputStream)
          throws IOException {
       hashCode();
       aOutputStream.writeInt(hash);
-      aOutputStream.writeByte(rowId.length);
+      aOutputStream.writeInt(rowId.length);
       aOutputStream.write(rowId,0,rowId.length);
-      aOutputStream.writeByte(tableId.length);
+      aOutputStream.writeInt(tableId.length);
       aOutputStream.write(tableId,0,tableId.length);
    }
 

diff --git a/src/main/java/com/yahoo/omid/tso/ThroughputMonitor.java b/src/main/java/com/yahoo/omid/tso/ThroughputMonitor.java
@@ -51,7 +51,7 @@ public void run() {
 
          long oldQueries = TSOHandler.queries;
          for (;;) {
-            Thread.sleep(1000);
+            Thread.sleep(10000);
 
             long endTime = System.currentTimeMillis();
             long newCounter = TSOHandler.getTransferredBytes();

diff --git a/src/main/native/Makefile b/src/main/native/Makefile
@@ -53,7 +53,7 @@ LIB=$(LIBDIR)/$(TSOCOMMITHASHMAP)
 all: $(LIB)
 
 $(LIB): $(OBJECTS)
-	-mkdir $(LIBDIR)
+	mkdir -p $(LIBDIR)
 	$(CC) $(LDFLAGS) $(OBJECTS) -o $@
 
 $(OBJECTS): $(SOURCES)

diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
@@ -17,7 +17,7 @@
 #
 ########################################################################
 
-log4j.rootLogger=ERROR,console
+log4j.rootLogger=WARN,console
 
 # Logging Threshold
 log4j.threshhold=ALL

diff --git a/src/test/java/com/yahoo/omid/TestUtils.java b/src/test/java/com/yahoo/omid/TestUtils.java
@@ -16,6 +16,8 @@
 
 package com.yahoo.omid;
 
+import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.Socket;
 import java.net.UnknownHostException;
@@ -71,5 +73,14 @@ public static void waitForSocketNotListening(String host, int port,
             LOG.info("Host " + host + ":" + port + " is up");
         }
     }
+
+    public static void delete(File f) throws IOException {
+        if (f.isDirectory()) {
+            for (File c : f.listFiles())
+                delete(c);
+        }
+        if (!f.delete())
+            throw new FileNotFoundException("Failed to delete file: " + f);
+    }
 
 }