Merge branch 'master' of https://github.com/geocam/geocamUtilWeb

.gitignore

@@ -1,4 +1,6 @@
 *.pyc
+*.swp
+*.swo
 dev.db
 local_settings.py
 media/ugc

geocamUtil/dotDict.py

@@ -0,0 +1,38 @@
+# __BEGIN_LICENSE__
+# Copyright (C) 2008-2010 United States Government as represented by
+# the Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# __END_LICENSE__
+
+from geocamUtil import anyjson as json
+
+
+def convertToDotDictRecurse(struct):
+    if isinstance(struct, dict):
+        for k, v in struct.iteritems():
+            struct[k] = convertToDotDictRecurse(v)
+        return DotDict(struct)
+    elif isinstance(struct, list):
+        return [convertToDotDictRecurse(elt) for elt in struct]
+    else:
+        return struct
+
+
+class DotDict(dict):
+    # At the moment this object exists pretty much solely to let you
+    # get and set elements in its __dict__ dictionary via dotted
+    # notation.  Someday it could do more.
+
+    # these are fields that must not be defined to avoid causing problems
+    # with Django
+    _badFields = ('prepare_database_save',)
+
+    def __repr__(self):
+        return json.dumps(self, sort_keys=True, indent=4)
+
+    def __getattr__(self, attr):
+        if attr in self._badFields:
+            raise KeyError(attr)
+        return self.get(attr, None)
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__

geocamUtil/gevent/util.py

@@ -9,11 +9,14 @@
 import fcntl
 import errno
 import re
+import logging
 
 import gevent
 from gevent import socket
 from gevent.queue import Queue
 
+END_OF_LINE = re.compile(r'\r\n|\n|\r')
+
 
 class LineParser(object):
     def __init__(self, handler, maxLineLength=None):
@@ -24,7 +27,7 @@ def __init__(self, handler, maxLineLength=None):
     def write(self, text):
         self._buf += text
         while 1:
-            m = re.search(r'\r\n|\n|\r', self._buf)
+            m = END_OF_LINE.search(self._buf)
             ind = m.end() if m else None
             if (ind == None
                 and self._maxLineLength is not None
@@ -46,12 +49,16 @@ def flush(self):
 END_OF_FILE = ('__EOF__',)
 
 
-def safeRead(fd, chunkSize):
+def safeRead(fd, chunkSize, label):
     try:
         chunk = os.read(fd, 1024)
     except OSError, ex:
         if ex[0] == errno.EAGAIN:
             return ''
+        if ex[0] == errno.EIO:
+            # what we see if the other end of the pipe dies uncleanly
+            return END_OF_FILE
+        logging.warning('safeRead exception, label=%s', label)
         raise
     if not chunk:
         return END_OF_FILE
@@ -64,7 +71,7 @@ def setNonBlocking(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, flags)
 
 
-def copyFileToQueue(f, q, maxLineLength=None):
+def copyFileToQueue(f, q, maxLineLength=None, label=None):
     """
     Given a file or file descriptor *f* (probably the output pipe from a
     subprocess), asynchronously copy lines from the file into gevent
@@ -81,7 +88,7 @@ def copyFileToQueue(f, q, maxLineLength=None):
 
         setNonBlocking(fd)
         while 1:
-            chunk = safeRead(fd, 1024)
+            chunk = safeRead(fd, 1024, label)
             if chunk is END_OF_FILE:
                 break
             if chunk:
@@ -102,7 +109,7 @@ def copyFileToQueue(f, q, maxLineLength=None):
         q.put(StopIteration)
 
 
-def queueFromFile(f, maxLineLength=None):
+def queueFromFile(f, maxLineLength=None, label=None):
     q = Queue()
-    gevent.spawn(copyFileToQueue, f, q, maxLineLength)
+    gevent.spawn(copyFileToQueue, f, q, maxLineLength, label)
     return q

geocamUtil/models/ExtrasDotField.py

@@ -8,37 +8,7 @@
 from django.core.exceptions import ValidationError
 
 from geocamUtil import anyjson as json
-
-
-def convertToDotDictRecurse(struct):
-    if isinstance(struct, dict):
-        for k, v in struct.iteritems():
-            struct[k] = convertToDotDictRecurse(v)
-        return DotDict(struct)
-    elif isinstance(struct, list):
-        return [convertToDotDictRecurse(elt) for elt in struct]
-    else:
-        return struct
-
-
-class DotDict(dict):
-    # At the moment this object exists pretty much solely to let you
-    # get and set elements in its __dict__ dictionary via dotted
-    # notation.  Someday it could do more.
-
-    # these are fields that must not be defined to avoid causing problems
-    # with Django
-    _badFields = ('prepare_database_save',)
-
-    def __repr__(self):
-        return json.dumps(self, sort_keys=True, indent=4)
-
-    def __getattr__(self, attr):
-        if attr in self._badFields:
-            raise KeyError(attr)
-        return self.get(attr, None)
-    __setattr__ = dict.__setitem__
-    __delattr__ = dict.__delitem__
+from geocamUtil.dotDict import DotDict, convertToDotDictRecurse
 
 
 class ExtrasDotField(models.TextField):

geocamUtil/pdf.py

@@ -4,6 +4,8 @@
 # All Rights Reserved.
 # __END_LICENSE__
 
+import tempfile
+import os
 
 class PdfConversionError(Exception):
     pass
@@ -18,4 +20,26 @@ def convertPdf(data,
     Pass in the raw binary PDF data. Returns the raw binary image data
     result after rasterization.
     """
-    raise NotImplementedError()
+
+    #write to temporary pdf file
+    tempInputFile = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False)
+    tempInputFile.seek(0)
+    tempInputFile.write(data)
+    tempInputFile.flush()
+
+    outputFileName = tempInputFile.name.replace('.pdf', '.png')
+
+    ret = os.system('convert -flatten %s %s > /dev/null' % (tempInputFile.name, outputFileName))
+
+    os.remove(tempInputFile.name)
+
+    if ret != 0 or os.path.isfile(outputFileName) == False:
+        raise PdfConversionError('Error found while converting pdf')
+
+    outputFile = open(outputFileName, 'r')
+    outputFileData = outputFile.read()
+
+    outputFile.close()
+    os.remove(outputFileName)
+
+    return outputFileData

geocamUtil/store.py

@@ -16,6 +16,8 @@
 import logging
 import json
 
+loggerG = logging.getLogger('geocamUtil.store')
+
 
 def encodeVal(val):
     return zlib.compress(pickle.dumps(val, pickle.HIGHEST_PROTOCOL), 9)
@@ -158,6 +160,7 @@ def purgeQueue(self):
                 cacheInfo.refCount -= 1
 
     def flushEntry(self, key):
+        loggerG.debug('flushEntry %s', key)
         cacheInfo = self.cacheInfo[key]
         if cacheInfo.dirty:
             val = self.cache[key]
@@ -167,28 +170,32 @@ def flushEntry(self, key):
             cacheInfo.dirty = False
 
     def evictEntry(self, key):
+        loggerG.debug('evictEntry %s', key)
         self.flushEntry(key)
         del self.cache[key]
         del self.cacheInfo[key]
 
     def evictLru(self):
         while len(self.cache) >= self.maxEntries:
             key = self.queue.popleft()
-            cacheInfo = self.cacheInfo[key]
-            if cacheInfo.refCount == 1:
-                self.evictEntry(key)
-            else:
-                cacheInfo.refCount -= 1
+            cacheInfo = self.cacheInfo.get(key, None)
+            if cacheInfo:
+                if cacheInfo.refCount == 1:
+                    self.evictEntry(key)
+                else:
+                    cacheInfo.refCount -= 1
 
     def sync(self):
-        for key in self.cache.keys():
-            self.flushEntry(key)
+        for key, cacheInfo in self.cacheInfo.iteritems():
+            if cacheInfo.dirty:
+                self.flushEntry(key)
 
     def __getitem__(self, key):
         if key in self.cache:
             self.markRead(key)
             return self.cache[key]
         else:
+            # hm... add to cache?
             return self.store[key]
 
     def __setitem__(self, key, val):

geocamUtil/storeTest.py

@@ -7,6 +7,7 @@
 import shutil
 import unittest
 import tempfile
+import logging
 
 from geocamUtil.store import FileStore, LruCacheStore
 
@@ -16,18 +17,26 @@ def storeTest(self, storeFactory):
         tempDir = tempfile.mkdtemp('-storeTestDir')
 
         store1 = storeFactory(tempDir)
-        store1['a'] = 1
-        store1['b'] = 2
+        for i in xrange(20):
+            store1[str(i)] = i
         store1.sync()
 
         store2 = storeFactory(tempDir)
-        self.assertEqual(store2['a'], 1)
-        self.assertEqual(store2['b'], 2)
+        for i in xrange(20):
+            self.assertEqual(store2[str(i)], i)
 
         shutil.rmtree(tempDir)
 
     def test_FileStore(self):
         self.storeTest(FileStore)
 
     def test_LruCacheStore(self):
-        self.storeTest(lambda path: LruCacheStore(FileStore(path), 100))
+        self.storeTest(lambda path: LruCacheStore(FileStore(path), 5))
+
+#    def test_LruEvict(self):
+#        pass
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    unittest.main()

geocamUtil/zmq/zclient.py

@@ -12,6 +12,10 @@
 from IPython.config.loader import Config
 from IPython.frontend.terminal.embed import InteractiveShellEmbed
 from IPython.lib.inputhook import inputhook_manager, stdin_ready
+try:
+    from django.conf import settings
+except ImportError:
+    settings = object()
 
 from geocamUtil.jsonConfig import loadConfig
 from geocamUtil.zmq.zerorpcClientProxy import ClientProxy
@@ -88,7 +92,7 @@ def main():
     import optparse
     parser = optparse.OptionParser('usage: %prog OPTIONS')
     parser.add_option('-p', '--ports',
-                      default='ports.json',
+                      default=getattr(settings, 'GEOCAM_UTIL_ZMQ_PORTS_PATH', 'ports.json'),
                       help='Path to ports config file [%default]')
     opts, args = parser.parse_args()
     if args:

geocamUtil/zmq/zerorpcClientProxy.py

@@ -11,7 +11,7 @@ class %(decoratedName)s(%(parentName)s):
 FUNC_TEMPLATE = """
     def %(name)s(self, %(signature)s):
         %(docstring)s
-        return self._client.%(name)s(%(signature)s)
+        return self._client.%(name)s(%(argNames)s)
 """
 
 
@@ -27,11 +27,15 @@ class MissingVal(object):
 MISSING = MissingVal()
 
 
-def declarationFromZerorpcInspectArg(arg):
+def nameFromArg(arg):
     name = arg['name']
     if isinstance(name, tuple):
         name = '(%s)' % ', '.join([n for n in name])
-    result = name
+    return name
+
+
+def declarationFromZerorpcInspectArg(arg):
+    result = nameFromArg(arg)
 
     defaultVal = arg.get('default', MISSING)
     if defaultVal is not MISSING:
@@ -66,13 +70,16 @@ def makeDecoratedProxy(cls, name, client):
 
         methods = meta['methods']
         for name, info in methods.iteritems():
-            argNames = [declarationFromZerorpcInspectArg(arg) for arg in info['args']]
-            signature = ', '.join(argNames[1:])
+            argSigs = [declarationFromZerorpcInspectArg(arg) for arg in info['args']]
+            signature = ', '.join(argSigs[1:])
+            argNames = ', '.join(nameFromArg(arg) for arg in info['args'][1:])
             docstring = repr(info.get('doc', '<unknown>'))
             src += (FUNC_TEMPLATE
                     % {'name': name,
                        'signature': signature,
+                       'argNames': argNames,
                        'docstring': docstring})
+        # print 'zerorpcClientProxy:', src
         code = compile(src, '<string>', 'single')
         evaldict = {'__name__': 'geocamUtil.zmq.zerorpcClientProxy',
                     'ClientProxy': ClientProxy}