New storage engine implementation #31 (#66)

Cleaning up archival moving sql package

time series buckets

Fixing bugs with refactor

and diskstorage engine

Validating and fixing unit tests

windows)

adding unit tests for persistent measurement

Adding concurrency fixes #31

Fixing recovery bug

Fixing broken recovery system due to missing offsets, fixing unit tests
for recovery, adding parallel recovery for measurements

Improving test converage

Fixing minor code issues

fixing potential concurrency issue

archiver/hdfs-archiver/src/test/java/com/srotya/sidewinder/archiver/hdfs/TestHDFSArchiver.java

@@ -30,8 +30,8 @@
 import com.srotya.sidewinder.core.storage.ArchiveException;
 import com.srotya.sidewinder.core.storage.Archiver;
 import com.srotya.sidewinder.core.storage.DataPoint;
-import com.srotya.sidewinder.core.storage.Reader;
 import com.srotya.sidewinder.core.storage.TimeSeriesBucket;
+import com.srotya.sidewinder.core.storage.compression.Reader;
 import com.srotya.sidewinder.core.storage.compression.byzantine.ByzantineWriter;
 import com.srotya.sidewinder.core.storage.mem.archival.TimeSeriesArchivalObject;
 
@@ -60,7 +60,8 @@ public void testHDFSArchive() throws IOException, ArchiveException {
 		conf.put(HDFSArchiver.HDFS_ARCHIVE_DIRECTORY, "target/test-hdfs-" + System.currentTimeMillis());
 		archiver.init(conf);
 		long ts = System.currentTimeMillis();
-		TimeSeriesBucket bucket = new TimeSeriesBucket("seriesId", ByzantineWriter.class.getName(), ts, false, new HashMap<>());
+		//TODO fix unit tests for HDFS Archiver
+		TimeSeriesBucket bucket = null;
 		for (int i = 0; i < 1000; i++) {
 			bucket.addDataPoint(ts + i * 1000, i);
 		}

cluster/src/main/java/com/srotya/sidewinder/cluster/ClusterResourceMonitor.java

@@ -19,6 +19,8 @@
 import java.util.Arrays;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import com.srotya.sidewinder.cluster.connectors.ClusterConnector;
 import com.srotya.sidewinder.core.storage.DataPoint;
@@ -30,7 +32,7 @@
 public class ClusterResourceMonitor {
 
 	private static final String DB = "_internal";
-//	private static Logger logger = Logger.getLogger(ClusterResourceMonitor.class.getName());
+	private static Logger logger = Logger.getLogger(ClusterResourceMonitor.class.getName());
 	private static final ClusterResourceMonitor INSTANCE = new ClusterResourceMonitor();
 	private StorageEngine storageEngine;
 	private ClusterConnector connector;
@@ -67,8 +69,7 @@ private void clusterMonitor() {
 			dp.setTags(Arrays.asList("controller"));
 			storageEngine.writeDataPoint(dp);
 		} catch (Exception e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
+			logger.log(Level.SEVERE, "Failed to monitor cluster", e);
 		}
 	}
 

cluster/src/main/java/com/srotya/sidewinder/cluster/routing/RoutingEngine.java

@@ -27,8 +27,8 @@
 import com.srotya.sidewinder.core.rpc.RawTimeSeriesBucket;
 import com.srotya.sidewinder.core.rpc.RawTimeSeriesBucket.Builder;
 import com.srotya.sidewinder.core.storage.StorageEngine;
+import com.srotya.sidewinder.core.storage.TimeSeries;
 import com.srotya.sidewinder.core.storage.TimeSeriesBucket;
-import com.srotya.sidewinder.core.storage.mem.TimeSeries;
 
 /**
  * Routing Engine provides the abstraction to stitch together the

core/pom.xml

@@ -139,6 +139,11 @@
       <version>2.2</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <version>1.4.0</version>
+    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-all</artifactId>