conflict resolve

#todo#

@@ -0,0 +1,22 @@
+1. Make the binder coverings:
+	 - Code listings/Data
+	 - Research Papers
+	 - Work Log
+	 - Graphs and Charts
+2. Code Listings
+	 - Figure out how many lines fit on one black and white page
+	 - Decide what parts to print
+
+3. Work Log
+	 - Write a work log based on the emails with Dusseau
+
+4. Graphs and Charts
+	 - Add legend to the graphs
+	 - print one out, see how it looks
+-------------------------------------------------------------------
+Things to print @ OfficeMax
+1. Graphs (did that)
+2. Graphs on the board (did that)
+3. Charts on the board (did that)
+4. Binder coverings
+5. 

.#todo

@@ -0,0 +1 @@
+dhaivat@dhaivat-GA-870A-UD3.15841:1392174308

.Rhistory

@@ -0,0 +1,6 @@
+dat = read.csv("analysis/latencies.csv", head=TRUE, sep=",")
+dat
+names(dat)
+dat$X.cached.latency.times.in.nanoseconds.
+boxplot(data)
+boxplot(dat)

.gitignore

@@ -0,0 +1,2 @@
+github.com/
+github.com/*

ROADMAP.md

@@ -0,0 +1,11 @@
+#ROADMAP
+
+## Issues to solve
+1. Implement another system that sits on the namenode and listens around
+for calls that change things on the filesystem. Should be independent of the 
+caching system entirely, i.e. a separate executable.
+2. Make the data caching work on multinode (might have to make a Writeable implementation)
+3. **Make PUT calls work**
+
+## New ideas
+1. Have to test the system on a popular usecase and see what happens

analysis/.#histogram.

@@ -0,0 +1 @@
+dhaivat@dhaivat-GA-870A-UD3.15841:1392174308

analysis/.Rhistory

@@ -0,0 +1,264 @@
+source("box-plot.r")
+source("box-plot.r")
+source("box-plot.r")
+dat <- read.csv("latencies.csv")
+dat
+dat[2]
+dat[2].max
+max(dat[2])
+source "box-plot.r"
+source("box-plot.r")
+source("box-plot.r")
+source("box-plot.r")
+source("box-plot.r")
+ls
+dat
+cached_latency
+cached_latencies
+cached_latency
+cached_latency = unlist(na.omit(da[1]))
+cached_latency = unlist(na.omit(dat[1]))
+no_cached_latency = unlist(na.omit(dat[2]))
+cached_latency
+hist(cached_latency, xlim=c(0, 1000000))
+cached_latency
+hist(cached_latency, xlim=c(0, 10000000))
+hist(cached_latency, xlim=c(0, 1000000))
+hist(cached_latency, xlim=c(0, 1000000), breaks=12)
+hist(cached_latency, xlim=c(0, 10000000), breaks=12)
+hist(cached_latency, breaks=12)
+hist(cached_latency, xlim=c(0, 10000000))
+cached_latency
+dat[1]
+plot(cached_latency)
+cached_latency
+plot(cached_latency, col='red')
+plot(cached_latency, col='red')
+plot(non_cached_latency, col='blue')
+plot(no_cached_latency, col='blue')
+plot(cached = cached_latency, non_cached = no_cached_latency)
+plot(cached_latency, col='red')
+hist(dat)
+plot(dat)
+plot(dat[1])
+plot(dat[2])
+plot(cached_latency, col='red')
+par(new=T)
+plot(no_cached_latency, col='blue')
+plot(cached_latency, col='red')
+lines(no_cached_latency, col='blue')
+points(no_cached_latency, col='blue')
+plot(cached_latency, col='red')
+lines(no_cached_latency, col='blue')
+plot(cached_latency, col='red')
+points(no_cached_latency, col='blue')
+title(ylab="access time (in nanoseconds)")
+title(ylab=NULL)
+range(cached_latency)
+hist(cached_latency, xlim=c(11034, 11034*10))
+hist(cached_latency, xlim=c(11034, 11034*10), breaks=200)
+hist(cached_latency, xlim=c(11034, 11034*10), breaks=200)
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+ls
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+ls
+source('histogram.r')
+hist(cached_latency)
+hist(cached_latency, buckets=200)
+hist(cached_latency, breaks=200)
+mean(cached_latency)
+median(cached_latency)
+hist(cached_latency, breaks=200, xlim=(0, 1000000))
+hist(cached_latency, breaks=200, xlim=c(0, 1000000))
+hist(cached_latency, breaks=200, xlim=c(0, 10000000))
+hist(cached_latency, breaks=200, xlim=c(0, 5000000))
+hist(cached_latency, breaks=20, xlim=c(0, 5000000))
+hist(cached_latency, breaks=2000, xlim=c(0, 5000000))
+hist(cached_latency, breaks=600, xlim=c(0, 5000000))
+hist(cached_latency, breaks=600, xlim=c(0, 2000000))
+hist(cached_latency, breaks=600, xlim=c(0, 1000000))
+hist(cached_latency, breaks=1000, xlim=c(0, 1000000))
+hist(cached_latency, breaks=1000, xlim=c(0, 1000000))
+hist(cached_latency, breaks=1000, xlim=c(0, 500000))
+hist(cached_latency, breaks=1000, xlim=c(0, 1000000))
+hist(cached_latency, breaks=1000, xlim=c(0, 800000))
+hist(cached_latency, breaks=100, xlim=c(0, 800000))
+hist(cached_latency, breaks=10000, xlim=c(0, 800000))
+hist(cached_latency, breaks=10000, xlim=c(0, 800000), color='red')
+hist(cached_latency, breaks=10000, xlim=c(0, 800000), col='red')
+hist(no_cached_latency, breaks=10000, col='blue')
+hist(cached_latency, breaks=10000, xlim=c(0, 800000), col='red')
+hist(no_cached_latency, breaks=10000, col='blue', add=T)
+mean(no_cache_latency)
+mean(no_cached_latency)
+hist(no_cached_latency, breaks=10000, col='red', xlim=c(0,1600000))
+hist(no_cached_latency, breaks=10000, col='red', xlim=c(0,16000000))
+hist(no_cached_latency, breaks=1000, col='red', xlim=c(0,16000000))
+hist(cached_latency, breaks=1000, col='red', add=T)
+hist(cached_latency, breaks=100, col='red', add=T)
+hist(cached_latency, breaks=100, col='blue', add=T)
+hist(no_cached_latency, breaks=10000, col='red', xlim=c(0,16000000))
+hist(cached_latency, breaks=1000, col='blue', add=T)
+hist(cached_latency, breaks=10000, col='blue', add=T)
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('histogram.r')
+source('box-plot.r
+')
+source('box-plot.r')
+ls
+source('point-plot.r')
+source('latencies.csv')
+source("point-plot.r")
+cat 
+cat("point-plot.r")
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r)
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plox.r')
+source("box-plot.r")
+dat <- read.csv("latencies.csv")
+cached = unlist(na.omit(dat[1]))
+noncached_data = unlist(na.omit(dat[2]))
+df <- data.frame(metacached = cached_data, non_metacached = noncached_data)
+ boxplot(df, ylim=c(0, 4000000), col=c('chocolate1', 'cornflowerblue'), xaxt="n", cex.main=1.5)
+title("metacached")
+par(cex=3.0)
+boxplot(df, ylim=c(0, 4000000), col=c('chocolate1', 'cornflowerblue'), xaxt="n", cex.main=1)
+help('par')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+source('box-plot.r')
+data_latencies <- read.csv("data/latencies.csv")
+cached = unlist(na.omit(data_latencies))
+cached = unlist(na.omit(data_latencies[1]))
+noncached = unlist(na.omit(data_latencies[2]))
+std(cached)
+sd(cached)
+sd(noncached)
+sd(noncached)/sd(cached)
+metadata_latencies <- read.csv("data/multinode_meta.csv")
+cached = unlist(na.omit(metadata_latencies[1]))
+noncached = unlist(na.omit(metadata_latencies[2]))
+sd(noncached)/sd(cached)
+dat
+dat <- read.csv("latencies.csv")
+df <- data.frame(dat)
+ggplot(df, aes=(x="Panthera (dls
+ggplot(df, aes=(x="Panthera (datacached) latency")) + geom_histogram()
+library(ggplot2)
+ggplot(df, aes=(x="Panthera (datacached) latency")) + geom_histogram()
+ggplot(df, aes(x="Panthera (datacached) latency")) + geom_histogram()
+ggplot(df, aes(x="Panthera (datacached) latency", data=cached)) + geom_histogram()
+ggplot(df, aes(x=cached)) + geom_histogram() + xlab("Panthera (datached) latency")
+ggplot(df, aes(x=cached)) + geom_histogram() + xlab("Panthera (datacached) latency") + title("Panthera (Datacached) Latency Distribution")
+ggplot(df, aes(x=cached)) + geom_histogram() + xlab("Panthera (datacached) latency") + ggtitle("Panthera (Datacached) Latency Distribution")
+pdf("datacached-latency-distribution-blackwhite.pdf")
+ggplot(df, aes(x=cached)) + geom_histogram() + xlab("Panthera (datacached) latency") + ggtitle("Panthera (Datacached) Latency Distribution")
+dev.off()
+dat <- read.csv("data/latencies.csv")
+dat[1]
+dat
+sd(unlist(na.omit(dat[1]))
+)
+sd(unlist(na.omit(dat[2]))
+)
+sd(unlist(na.omit(dat[2]))
+)
+dat <- read.csv("data/old_meta_latencies.csv")
+sd(unlist(na.omit(dat[1]))
+)
+sd(unlist(na.omit(dat[2]))
+)
+dat <- read.csv("data/multinode_meta.csv")
+sd(unlist(na.omit(dat[1]))
+)
+dat <- read.csv("data/multinode_meta.csv")
+sd(unlist(na.omit(dat[2]))
+)
+ls
+source('box-plot.r')
+source('box-plot.r')

analysis/box-plot.r

@@ -0,0 +1,17 @@
+#read in the data
+dat <- read.csv("latencies.csv", head = TRUE, sep=",")
+
+cached_data = unlist(na.omit(dat[1]))
+noncached_data = unlist(na.omit(dat[2]))
+
+df <- data.frame(metacached = cached_data, non_metacached = noncached_data)
+pdf("boxplot.pdf", width=9.5, height=7)
+par(mar=c(5, 4.5, 4, 2))
+boxplot(df, ylim=c(0, 40000000), col=c('chocolate1', 'cornflowerblue'), xaxt="n", cex.main=1.5, whisklwd=4, staplelwd=4, outlwd=2, outcex=1.2)
+par(cex.main=1.9)
+title("Metacached (Panthera) vs Non-metacached (Status Quo)\nMultinode")
+par(cex.axis=1.8)
+axis(1, at=1:2, labels=c("metacached (Panthera)", "non-metacached"))
+par(cex.lab=1.8)
+title(ylab = "Access Time (nanoseconds, lower is better)")
+dev.off()

analysis/box-plot.r~

@@ -0,0 +1,11 @@
+#read in the data
+dat <- read.csv("latencies.csv", head = TRUE, sep=",")
+
+cached_data = unlist(na.omit(dat[1]))
+noncached_data = unlist(na.omit(dat[2]))
+
+df <- data.frame(cached = cached_data, noncached = noncached_data)
+png("boxplot.png")
+boxplot(df)
+title("Panthera (cached) vs Status Quo (non-cached) - Data")
+