major service overhaul, start of ui integration

logjam/client.py

@@ -15,26 +15,29 @@ def connect(self, host=settings.HOST, port=settings.PORT, timeout=settings.TIMEO
             self.socket.connect((host,  port))
         except Exception, e:
             if e.errno == 61:
-                print 'Failed to connect'
-            self.socket = None
-
+                print >>sys.stderr, 'Failed to connect'
+            raise
+        
     def close(self):
         return self.socket.close()
 
     def all(self):
         for sha in self.run('LIST').splitlines():
-            yield sha
+            yield sha.rstrip()
 
     def get_first(self):
         for sha in self.all():
             return self.get(sha)
 
     def get_all(self):
         for sha in self.all():
-            yield self.get(sha.strip())
+            yield self.get(sha)
 
     def get(self, sha):
         return deserialize(self.run('GET%s' % sha))
+
+    def dump(self):
+        self.run('DUMP')
 
     def run(self, command):
         self.connect()
@@ -54,10 +57,18 @@ def send_exception(self, request, exc_info=None):
         self.socket.send(sha)
 
         if self.socket.recv(1) == '\x00':
-            print 'Ignoring %s' % sha
+            print >>sys.stderr, 'Ignoring %s' % sha
             self.close()
             return
 
-        print 'Sending %s' % sha
-        self.socket.send(serialize(request2dict(request, exception, sha)))
+        print >>sys.stderr, 'Sending %s' % sha
+        try:
+            d = request2dict(request, exception, sha)
+        except Exception, e:
+            print >>sys.stderr, 'Conversion error: %s'%e
+
+        try:
+            self.socket.send(serialize(d))
+        except Exception, e:
+            print >>sys.stderr, 'Client error %s' % e
         self.close()

logjam/handlers.py

@@ -18,10 +18,19 @@ def handle_uncaught_exception(self, request, resolver, exc_info):
             from django.views import debug
             return debug.technical_500_response(request, *exc_info)
 
-        # Send exception to logjam server
-        from logjam.client import send_exception
-        send_exception(request, exc_info)
-
+
+        from logjam.client import Client
+        Client().send_exception(request, exc_info)
+
+        #from django.core.mail import mail_admins
+        #subject = 'Error (%s IP): %s' % ((request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), request.path)
+        #try:
+        #    request_repr = repr(request)
+        #except:
+        #    request_repr = "Request repr() unavailable"
+        #message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
+        #mail_admins(subject, message, fail_silently=True)
+
         # Return an HttpResponse that displays a friendly error message.
         callback, param_dict = resolver.resolve500()
         return callback(request, **param_dict)

logjam/management/commands/logserver.py

@@ -1,52 +1,68 @@
 from SocketServer import BaseRequestHandler, ThreadingMixIn, TCPServer
 from django.core.management.base import BaseCommand
-from django.core.cache import get_cache
 from logjam import settings
 from logjam.util import *
 from pprint import pprint
 from atexit import register
+from datetime import datetime
 import fcntl
 import os
 import sys
 
 
-cache = get_cache(settings.CACHE)
-ctrl = '%s%s' % (settings.PREFIX, 'ctrl')
-
 class ExceptionHandler(BaseRequestHandler):
+    def all(self):
+        return cache.get(ctrl, ())
+
+    def dump(self):
+        pickle.dump(map(self.get, self.all()), open(datetime.now().strftime(settings.DUMP_FILE),'w'), -1)
+
+
+    def get(self, sha):
+        return cache.get('%s%s' % (settings.PREFIX, sha), '(dp0\n.')
+
     def handle(self):
-        sha = self.request.recv(40)
-        if is_hex(sha):
-            key = '%s%s' % (settings.PREFIX, sha)
-            cached = cache.get(key)
-            if cached:
-                print 'Ignoring %s' % sha
-                self.request.send('\x00')
-            else:
-                print 'Saving %s' % sha
-                self.request.send('\x01')
-                content = ''
-                while 1:
-                    bit = self.request.recv(1024)
-                    if not bit: break
-                    content += bit 
-                cache.set(key, content)
-                cache.set(ctrl, cache.get(ctrl, ()) + (sha,))
-                log_report(deserialize(content))
-        elif sha.startswith('LIST'):
-            self.request.send('\n'.join(cache.get(ctrl, ())))
-        elif sha.startswith('GET'):
-            sha += self.request.recv(3)
-            sha = sha[3:]
-            print 'Getting %s' % sha
-            self.request.send(cache.get('%s%s' % (settings.PREFIX, sha), '(dp0\n.'))
+        try:
+            data = self.request.recv(40)
+            if sha_re.match(data):
+                key = '%s%s' % (settings.PREFIX, data)
+                cached = cache.get(key)
+                if cached is None:
+                    print >>settings.LOG_FILE, 'Saving %s' % data
+                    self.request.send('\x01')
+                    content = ''
+                    while 1:
+                        bit = self.request.recv(1024)
+                        if not bit: break
+                        content += bit
+                    if not content:
+                        print >>settings.ERROR_FILE, 'No content to save'
+                        return
+                    cache.set(key, content, 36000)
+                    cache.set(ctrl, cache.get(ctrl, ()) + (data,), 36000)
+                    #log_report(deserialize(content))
+                else:
+                    print >>settings.LOG_FILE, 'Ignoring %s' % data
+                    self.request.send('\x00')
+            elif data.startswith('DUMP'):
+                self.dump()
+            elif data.startswith('LIST'):
+                self.request.send('\n'.join(self.all()))
+            elif data.startswith('GET'):
+                data += self.request.recv(3)
+                data = data[3:]
+                self.request.send(self.get(data))
+            elif data.startswith('DEL'):
+                data += self.request.recv(3)
+                data = data[3:]
+                cache.delete(data)
+        except Exception, e:
+            import traceback
+            print >>settings.ERROR_FILE, 'Server Error: %s' % traceback.format_exc()
 
-from OpenSSL import SSL
-import socket
-
 class LogJamServer(ThreadingMixIn, TCPServer):
     pass
-
+    #from OpenSSL import SSL
     #def __init__(self, server_address, HandlerClass):
     #    TCPServer.__init__(self, server_address, HandlerClass)
     #    ctx = SSL.Context(SSL.SSLv23_METHOD)
@@ -93,7 +109,7 @@ def handle(self, *args, **options):
 
         null_descriptor = open(getattr(os, 'devnull', '/dev/null'), 'rw')
         if not settings.DEBUG:
-            for descriptor in (sys.stdin, sys.stdout, sys.stderr):
+            for descriptor in (sys.stdin, sys.stdout, settings.LOG_FILE):
                 descriptor.close()
                 descriptor = null_descriptor
 
@@ -111,5 +127,9 @@ def handle(self, *args, **options):
         pidfile.write('%s' %(os.getpid()))
         pidfile.flush()
 
-        register(server.socket.close)
-        server.serve_forever()
+        #register(server.socket.close)
+        try:
+            server.serve_forever()
+        except:
+            #server.shutdown()
+            sys.exit(0)

logjam/middleware.py

@@ -1,10 +1,12 @@
-from client import send_exception
-from util import format_exception
+from client import Client
 
 
 class LogJamMiddleware(object):
     def process_exception(self, request, exception):
-        send_exception(request)
+        try:
+            Client().send_exception(request)
+        except:
+            print 'asdf'
 
 """
 http_load -parallel 20 -seconds 30 urls

logjam/settings.py

@@ -2,15 +2,17 @@
 import sys
 
 HOST = getattr(settings, 'LOGJAM_HOST', 'localhost')
-PORT = getattr(settings, 'LOGJAM_PORT', 8001)
+PORT = getattr(settings, 'LOGJAM_PORT', 8003)
 TIMEOUT = getattr(settings, 'LOGJAM_TIMEOUT', 5)
 CACHE = getattr(settings, 'LOGJAM_CACHE_BACKEND', 'locmem:///')
 LOG_FILE = getattr(settings, 'LOGJAM_LOG_FILE', sys.stdout)
+ERROR_FILE = getattr(settings, 'LOGJAM_ERROR_FILE', sys.stderr)
+#'<ExceptionRequest\nGET:%(GET)s,\nPOST:%(POST)s,\nCOOKIES:%(COOKIES)s,\nMETA:%(META)s>\n\n%(exception)s\n')
 LOG_FORMAT = getattr(settings, 'LOGJAM_LOG_FORMAT', '%(host)s - [%(timestamp)s] "%(method)s %(path)s" \n%(exception)s')
-                     #'<ExceptionRequest\nGET:%(GET)s,\nPOST:%(POST)s,\nCOOKIES:%(COOKIES)s,\nMETA:%(META)s>\n\n%(exception)s\n')
+ARCHIVE_PATH = getattr(settings, 'LOGJAM_ARCHIVE_PATH', '/tmp/logjam/%Y/%m/%d')
 PREFIX = getattr(settings, 'LOGJAM_CACHE_PREFIX', 'logjam_')
-UNIQUE = getattr(settings, 'LOGJAM_UNIQUE', False)
+UNIQUE = getattr(settings, 'LOGJAM_UNIQUE', True)
 PIDFILE = getattr(settings, 'LOGJAM_PIDFILE', '/tmp/logjam.pid')
 DEBUG = getattr(settings, 'LOGJAM_DEBUG', True)
-DAEMONIZE = getattr(settings, 'LOGJAM_DAEMONIZE', False)
+DAEMONIZE = getattr(settings, 'LOGJAM_DAEMONIZE', True)
 COMPRESS = getattr(settings, 'LOGJAM_COMPRESSION', 5)

logjam/templates/logjam/admin.html

@@ -0,0 +1,73 @@
+{% extends 'admin/base.html' %}
+{% load native %}
+
+{% block title %}{{ title }} | LogJam Admin {% endblock %}
+{% block branding %}
+<h1 id="site-name">LogJam Admin</h1>
+{% endblock %}
+
+{% block extrahead %}
+<!--<link rel="stylesheet" href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.7.2/themes/ui-lightness/jquery-ui.css" type="text/css" media="all" />-->
+<!--<link rel="stylesheet" href="http://static.jquery.com/ui/css/demo-docs-theme/ui.theme.css" type="text/css" media="all" />-->
+<!--<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script>-->
+<!--<script src="http://ajax.googleapis.com/ajax/libs/jqueryui/1.7.2/jquery-ui.min.js" type="text/javascript"></script>-->
+
+<link type="text/css" href="{{ MEDIA_URL }}css/custom-theme/jquery-ui-1.7.2.custom.css" rel="stylesheet" />	
+<script type="text/javascript" src="{{ MEDIA_URL }}js/jquery-1.3.2.min.js"></script>
+<script type="text/javascript" src="{{ MEDIA_URL }}js/jquery-ui-1.7.2.custom.min.js"></script>
+
+<script type="text/javascript">
+	$(function() {
+		$("#accordion").accordion();
+                $(".tabs").tabs();
+                $("#datepicker1").datepicker();
+                $("#datepicker2").datepicker();
+	});
+	</script>
+
+<style>{% highlight_style style=friendly %}</style>
+{% endblock %}
+
+{% block content %}
+<form>
+<p><b>Filter Date</b>: from <input id="datepicker1" type="text" name="from"> to <input id="datepicker2" type="text" name="to">
+<input type="submit" value="Filter"/></p>
+</form>
+
+<div id="accordion">
+{% for e in errors %}
+    <h3><a href="#">{{ e.method }} {{ e.path }} - {% last_line e.exception %}</a></h3>
+    <div class="tabs" style="height: 300px; overflow: auto;">
+	<ul>
+            <li><a href="#err{{ forloop.counter }}-info">Info</a></li>
+            <li><a href="#err{{ forloop.counter }}-exc">Exception</a></li>
+            <li><a href="#err{{ forloop.counter }}-req">Request</a></li>
+	</ul>
+        <div id="err{{ forloop.counter }}-info">
+            <table>
+                <tr><th>Reported</th><td>{{ e.timestamp|date }} {{ e.timestamp|time }}</td></tr>                
+                <tr><th>Client</th><td>{{ e.META.REMOTE_ADDR }} - {{ e.META.HTTP_USER_AGENT }}</td></tr>
+                <tr><th>Host</th><td>{{ e.host }}</td></tr>
+                <tr><th>Method</th><td>{{ e.method }}</td></tr>
+                <tr><th>Path</th><td>{{ e.path }}</td></tr>
+                <tr><th>Exception</th><td>{% last_line e.exception %}</td></tr>
+            </table>
+        </div>
+        <div id="err{{ forloop.counter }}-exc">
+            {% highlight e.exception python %}
+        </div>
+        <div id="err{{ forloop.counter }}-req">
+            <table>
+                {% if e.GET %}<tr><th>GET</th><td><pre>{{ e.GET|pprint }}</pre>{% endif %}
+                {% if e.POST %}<tr><th>POST</th><td><pre>{{ e.POST|pprint }}</pre></td>{% endif %}
+                {% if e.COOKIES %}<tr><th>COOKIES</th><td><pre>{{ e.COOKIES|pprint }}</pre></td>{% endif %}
+                {% if e.FILES %}<tr><th>FILES</th><td><pre>{{ e.FILES|pprint }}</pre></td>{% endif %}
+                <tr><th>META</th><td><pre>{{ e.META|pprint }}</pre></td>
+            </table>
+        </div>
+    </div>
+{% endfor %}
+</div>
+
+{% endblock %}
+

logjam/tests.py

@@ -1,14 +1,16 @@
-import sys
-import datetime
+import sys,os
+from datetime import datetime
 from django.test import TestCase
 from django.http import HttpRequest
 from client import Client
 from mapreduce import WordCounter, Mailer
+import settings
+import pickle
 
 class MyReq(HttpRequest):
     def __init__(self):
         self.method = 'GET'
-        self.timestamp = datetime.datetime(2009, 12, 11, 14, 4, 59, 257218)
+        self.timestamp = datetime(2009, 12, 11, 14, 4, 59, 257218)
         self.path = '/'
         self.GET = {}
         self.POST = {}
@@ -77,6 +79,10 @@ def test_words(self):
         from operator import itemgetter
         self.assertEqual(l, sorted(l, key=itemgetter(1)))
 
+    def test_zdump(self):
+        self.client.dump()
+        print pickle.load(open(datetime.now().strftime(settings.DUMP_FILE)))
+
     def test_za(self):
         self.assert_('is_ajax' in self.client.get_first())
 

logjam/urls.py

@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('logjam.views',
+     (r'^$', 'admin'),
+     (r'get/(P?<sha>[a-f0-9]{40})$', 'archive'),
+)
+