Major: Add HTTP Server for serving content (fixes #2)

restructure files
Rework Authentication core.

gui/static/index.html → gui/index.html

@@ -67,10 +67,12 @@ <h3>HoneyProxy</h3>
 <td class=contenttype>
 	<%- getContentType() %></td>
 <td class=size>
-	<div title="<%- getRawContentSize() %> Bytes">
-		<%- getContentSize() %>
+	<div class=hidden><%- /*table sorter */ getResponseContentLength() %></div>
+	<div title="<%- getResponseContentLength() %> Bytes">
+		<%- getResponseContentLengthFormatted() %>
 	</div></td>
 <td class=time>
+	<div class=hidden><%- /*table sorter */ get("request").timestamp %></div>
 	<div class=timestamp title="Timestamp: <%- get("request").timestamp %>">
 	<%- getDate().toLocaleTimeString() %>, <%- ("0"+getDate().getDay()).slice(-2)+"."+("0"+getDate().getMonth()).slice(-2) %>.<br>
 	</div>	
@@ -84,9 +86,8 @@ <h3>HoneyProxy</h3>
 <div title="Pop out" class=popoutbutton></div>
 <div class=content>
 	<h2 class=title><%- getFilename() %></h2>
-	<% if(hasResponseContent()) print(getPreview());
-	   else print("No response content.");
-	 %>
+	<% if(hasResponseContent()) return getPreview(); 
+		else print("No response content."); %>
 </div>
 </script>
 <script id=template-header type="text/html">
@@ -150,6 +151,7 @@ <h2 class=title><%- getFilename() %></h2>
 <script src="./js/backbone.marionette.js"></script>
 
 <script src="./js/honeyproxy/honeyproxy.js"></script>
+<script src="./js/honeyproxy/Config.js"></script>
 <script src="./js/honeyproxy/utilities.js"></script>
 <script src="./js/honeyproxy/websocket.js"></script>
 <script src="./js/honeyproxy/Flow.js"></script>
@@ -162,6 +164,6 @@ <h2 class=title><%- getFilename() %></h2>
 <script src="./js/honeyproxy/TableSorter.js"></script>
 <script src="./js/honeyproxy/MainLayout.js"></script>
 <script src="./js/honeyproxy/DetailView.js"></script>
-<script src="./js/honeyproxy/PopOut.js"></script>
+<script src="./js/honeyproxy/popOut.js"></script>
 </body>
 </html>

gui/js/honeyproxy/Config.js

@@ -0,0 +1,16 @@
+(function(){
+
+
+	Config = function(data){
+		this.storage = {}; //localStorage might leak sensitive information
+		$.extend(this.storage, data);
+	};
+	Config.prototype.get = function(id){
+		return this.storage[id];
+	};
+	Config.prototype.set = function(id,val){
+		this.storage[id] = val;
+	}
+	HoneyProxy.config = new Config(
+			JSON.parse(decodeURIComponent(location.hash).replace("#","")));
+})();

gui/static/js/honeyproxy/Flow.js → gui/js/honeyproxy/Flow.js

@@ -22,28 +22,45 @@ HoneyProxy.Flow = Backbone.Model.extend({
 	getStatusCode: function(){
 		return this.get("response").code;
 	},
-	getContent: function(){
-		return this.get("response").content;
-	},
-	getRawContentSize: function(){
-		return this.get("response").content.length
-	},
-	getContentSize: function(){
-		if(!this.has("contentSize"))
+	getResponseContentURL: function(action){
+		var url = HoneyProxy.config.get("content")
+			+"/"+this.get("id")
+			+"/response/"+action
+			+"?"+$.param(
+					{"auth":HoneyProxy.config.get("auth")});
+		return url;
+	},
+	getResponseContentDownloadURL: function(){
+		return this.getResponseContentURL("attachment");
+	},
+	getResponseContentViewURL: function(){
+		return this.getResponseContentURL("inline");
+	},
+	getResponseContent: function(callback){
+		if(this.hasResponseContent())
+			return $.get(this.getResponseContentViewURL(),callback);
+		else
+			return callback("");
+	},
+	getResponseContentLength: function(){
+		return this.get("response").contentLength
+	},
+	getResponseContentLengthFormatted: function(){
+		if(!this.has("responseContentLength"))
 		{
 			var prefix = ["B","KB","MB","GB"];
-			var size = this.getRawContentSize();;
+			var size = this.getResponseContentLength();
 			while(size > 1024 && prefix.length > 1){
 				prefix.shift();
 				size = size / 1024;
 			}
-			this.set("contentSize",Math.floor(size)+prefix.shift());
+			this.set("responseContentLength",Math.floor(size)+prefix.shift());
 		}
-		return this.get("contentSize");
+		return this.get("responseContentLength");
 
 	},
 	hasResponseContent: function(){
-		return this.getRawContentSize() > 0;
+		return this.getResponseContentLength() > 0;
 	},
 	getContentType: function(){
 		if(!this.has("contentType"))

gui/static/js/honeyproxy/flows/CssFlow.js → gui/js/honeyproxy/flows/CssFlow.js

@@ -5,6 +5,8 @@ HoneyProxy.CSSFlow = HoneyProxy.DocumentFlow.extend({
 }, {matches: function(data){
 	if(data.contentType)
 		return !!data.contentType.match(/css/i);
+	else if(data.path)
+		return !!data.path.match(/\.css$/i);
 	return false;
 }});
 HoneyProxy.flowModels.unshift(HoneyProxy.CSSFlow);

gui/static/js/honeyproxy/flows/DocumentFlow.js → gui/js/honeyproxy/flows/DocumentFlow.js

@@ -3,11 +3,17 @@ HoneyProxy.DocumentFlow = HoneyProxy.Flow.extend({
 		return "document";
 	},
 	getPreview: function(){
-		return "<pre>Content: "+_.escape(this.getContent())+" </pre>";
+		var $pre = $("<pre>");
+		$pre.text("Loading...");
+		this.getResponseContent(function(data){
+			console.log(arguments)
+			$pre.text("Content: \n"+data);
+		});
+		return $pre[0];
 	}
 }, {matches: function(data){
 	if(data.contentType)
-		return !!data.contentType.match(/text/i);
+		return !!data.contentType.match(/application|text/i);
 	return false;
 }});
 HoneyProxy.flowModels.unshift(HoneyProxy.DocumentFlow);

gui/static/js/honeyproxy/flows/ImageFlow.js → gui/js/honeyproxy/flows/ImageFlow.js

@@ -3,9 +3,9 @@ HoneyProxy.ImageFlow = HoneyProxy.Flow.extend({
 		return "image";
 	},
 	getPreview: function(){
-		var contentType = this.getContentType() || "image/"+this.getFilename().split(".").pop();
-		contentType = contentType.replace(/[^a-zA-Z0-9\/]/g,"");
-		return '<img alt="preview" src="data:'+contentType+';base64,'+_.escape(window.btoa(this.getContent()))+'">';
+		//var contentType = this.getContentType() || "image/"+this.getFilename().split(".").pop();
+		//contentType = contentType.replace(/[^a-zA-Z0-9\/]/g,"");
+		return '<img src="'+this.getResponseContentViewURL()+'" alt="preview" >';
 	}
 
 }, {matches: function(data){

gui/static/js/honeyproxy/websocket.js → gui/js/honeyproxy/websocket.js

@@ -22,10 +22,9 @@ HoneyProxy.websocket = {
 		this.ws.send(JSON.stringify(jsonMsg));
 	},
 	initialize: function(){
-		var conn = this.getConnectionData();
-		this.ws = new WebSocket(conn.ws);
+		this.ws = new WebSocket(HoneyProxy.config.get("ws"));
 		this.ws.onopen = function(e){
-			HoneyProxy.websocket.send({action:"auth",key:conn.auth});
+			HoneyProxy.websocket.send({action:"auth",key:HoneyProxy.config.get("auth")});
 			HoneyProxy.log("Connection etablished");
 		};
 		this.ws.onmessage = this.onmessage;
@@ -51,8 +50,5 @@ HoneyProxy.websocket = {
 		}
 
 		HoneyProxy.log(e);
-	},
-	getConnectionData: function(){
-		return JSON.parse(decodeURIComponent(location.hash).replace("#",""));
 	}
 };

honeyproxy.py

@@ -17,7 +17,7 @@
 
 
 
-import gui.session
+import libhproxy.gui.session as session
 import sys, json, urllib
 sys.path.append("./mitmproxy") #git submodules "hack"
 
@@ -28,14 +28,13 @@
 from twisted.web.server import Site 
 from twisted.web.static import File
 from twisted.internet import reactor, task
+from twisted.web.resource import Resource
 
-from libhproxy.websockets import WebSocketsResource
-from libhproxy import proxy as hproxy, cmdline as hcmdline, version
+#from libhproxy.websockets import WebSocketsResource
+from libhproxy import proxy as hproxy, cmdline as hcmdline, version, content
 from libhproxy.honey import HoneyProxy
 
-from autobahn.websocket import WebSocketServerFactory, \
-                               WebSocketServerProtocol, \
-                               listenWS
+from autobahn.websocket import listenWS
 
 def main():
 
@@ -69,26 +68,34 @@ def main():
             print >> sys.stderr, "%(name)s:" % version.NAME, v.args[0]
             sys.exit(1)
 
+    HoneyProxy.setAuthKey(options.apiauth)
+
     #set up HoneyProxy GUI
-    guiSessionFactory = gui.session.GuiSessionFactory("ws://localhost:"+str(options.apiport),options.apiauth)
+    guiSessionFactory = session.GuiSessionFactory("ws://localhost:"+str(options.apiport))
 
+    #WebSocket
     listenWS(guiSessionFactory)
-
     #websocketRes = WebSocketsResource(guiSessionFactory)
     #reactor.listenTCP(options.apiport, Site(websocketRes))
-    reactor.listenTCP(options.guiport, Site(File("./gui/static")))    
 
+    root = Resource()
+    root.putChild("app",File("./gui"))
+    root.putChild("files", content.ContentAPI())
+    reactor.listenTCP(options.guiport, Site(root))  
+
     #HoneyProxy Master
     p = hproxy.HoneyProxyMaster(server, dumpoptions, filt, guiSessionFactory)
     HoneyProxy.setProxyMaster(p)
     p.start()
 
     wsURL = "ws://localhost:"+str(options.apiport)
+    contentURL = "http://localhost:"+str(options.guiport)+"/files"
     urlData = urllib.quote(json.dumps({
           "ws": wsURL,
-          "auth": guiSessionFactory.authKey
+          "auth": HoneyProxy.getAuthKey(),
+          "content": contentURL
           }))
-    guiURL = "http://localhost:"+str(options.guiport)+"/#"+urlData
+    guiURL = "http://localhost:"+str(options.guiport)+"/app#"+urlData
 
     if not options.nogui:
         #start gui
@@ -97,7 +104,8 @@ def main():
     else:
         print "GUI: "+guiURL
         print "WebSocket API URL: "+wsURL
-        print "Auth key: "+ guiSessionFactory.authKey
+        print "HTTP Content API Root: "+contentURL
+        print "Auth key: "+ HoneyProxy.getAuthKey()
 
     #run!
     l = task.LoopingCall(p.tick)

libhproxy/content.py

@@ -0,0 +1,51 @@
+from twisted.web.resource import Resource  
+from libhproxy.honey import HoneyProxy
+#serve request content via HTTP
+class ContentAPI(Resource):
+    isLeaf = True
+    def render_GET(self, request):
+        if self.isAuthenticated(request):
+            try:
+                if(len(request.postpath) != 3):
+                    raise Exception("invalid parameter length")
+                flow = HoneyProxy.getProxyMaster().getFlowCollection().getFlow(int(request.postpath[0]))
+                isResponse = request.postpath[1] == "response"
+
+                obj = getattr(flow,request.postpath[1])
+
+                isView = request.postpath[2] == "inline"
+                if (isResponse):
+                    #add important headers from original request
+                    headers = ["Content-Type","Content-Encoding","Transfer-Encoding"]
+                    for h in headers:
+                        if(h in obj.headers):
+                            request.setHeader(h,obj.headers.get(h)[0])
+
+                    #this would fail on 301 redirects
+                    #fix responsecode      
+                    #request.setResponseCode(obj.code)
+
+                    #fix content disposition for attachment download
+                    cdisp = obj.headers.get("Content-Disposition")
+                    if(cdisp == None):
+                        #do minimal file name guessing
+                        cdisp = 'inline; filename="'+flow.request.path.split("?")[0].split("/")[-1]+'"'
+                    if isView:
+                        request.setHeader("Content-Disposition",cdisp.replace("attachment", "inline"))
+                    else:
+                        request.setHeader("Content-Disposition",cdisp.replace("inline", "attachment"))
+
+                return obj.content
+            except Exception as e:
+                print e
+                return "<html><body>Invalid request.</body></html>"         
+        else:
+            request.setResponseCode(402)
+            return "<html><body>Please authenticate.</body></html>"      
+
+
+    def isAuthenticated(self,request):
+        try:
+            return request.args["auth"][0] == HoneyProxy.getAuthKey()
+        except:
+            return False

libhproxy/flowcollection.py

@@ -1,38 +1,35 @@
-from libmproxy import encoding
-from libmproxy.flow import ODictCaseless
 class FlowCollection:
     def __init__(self):
-        #self._flows = []
-        self._flows_json = []
-
-
-#    def getFlow(self,i):
-#        if(i < len(self._flows)):
-#            return self._flows[i]
-#        return None
+        self._flows_serialized = []
+        self._flows = []
 
     def getLastFlow(self):
-        return self._flows_json[-1]
-    
-    def getFlowsAsJSON(self):
-        return self._flows_json
+        return self._flows_serialized[-1]
+
+    def getFlow(self,flowId):
+        return self._flows[flowId]
 
-    def getFlowsAsSingleJSON(self):
-        return self._flows_json
-        #return ''.join(["[",','.join(self._flows_json),"]"])
+    def getFlowsSerialized(self):
+        return self._flows_serialized
 
     def addFlow(self, flow):
         flowRepr = flow._get_state()
-        flowRepr["id"] = len(self._flows_json)
+        flowRepr["id"] = len(self._flows_serialized)
 
-
-        enc = flow.response.headers.get("content-encoding")
-        if enc and enc[0] != "identity":
-            decoded = encoding.decode(enc[0], flow.response.content)
-            if decoded:
-                flowRepr["response"]["content"] = decoded
+        #remove content out of the flowRepr
+        for i in ["request","response"]:
+            flowRepr[i]["contentLength"] = len(flowRepr[i]["content"])
+            del flowRepr[i]["content"]
+
+        #store unencoded
+        #from libmproxy import encoding
+        #enc = flow.response.headers.get("content-encoding")
+        #if enc and enc[0] != "identity":
+        #    decoded = encoding.decode(enc[0], content)
+        #    if decoded:
+        #        content = decoded
 
-        #self._flows_json.append(json.dumps(flowRepr,ensure_ascii=None))
-        self._flows_json.append(flowRepr)
-        return len(self._flows_json)-1
+        self._flows.append(flow)
+        self._flows_serialized.append(flowRepr)
+        return len(self._flows_serialized)-1