Permalink
Browse files

Merge branch 'master' of https://github.com/geocam/geocamUtilWeb

  • Loading branch information...
2 parents d58dd6e + f53aeef commit 48c541627496a75014bdfa9d718a3f4153341b8d @deleted deleted committed Jan 23, 2013
View
@@ -1,4 +1,6 @@
*.pyc
+*.swp
+*.swo
dev.db
local_settings.py
media/ugc
View
@@ -0,0 +1,38 @@
+# __BEGIN_LICENSE__
+# Copyright (C) 2008-2010 United States Government as represented by
+# the Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# __END_LICENSE__
+
+from geocamUtil import anyjson as json
+
+
+def convertToDotDictRecurse(struct):
+ if isinstance(struct, dict):
+ for k, v in struct.iteritems():
+ struct[k] = convertToDotDictRecurse(v)
+ return DotDict(struct)
+ elif isinstance(struct, list):
+ return [convertToDotDictRecurse(elt) for elt in struct]
+ else:
+ return struct
+
+
+class DotDict(dict):
+ # At the moment this object exists pretty much solely to let you
+ # get and set elements in its __dict__ dictionary via dotted
+ # notation. Someday it could do more.
+
+ # these are fields that must not be defined to avoid causing problems
+ # with Django
+ _badFields = ('prepare_database_save',)
+
+ def __repr__(self):
+ return json.dumps(self, sort_keys=True, indent=4)
+
+ def __getattr__(self, attr):
+ if attr in self._badFields:
+ raise KeyError(attr)
+ return self.get(attr, None)
+ __setattr__ = dict.__setitem__
+ __delattr__ = dict.__delitem__
@@ -9,11 +9,14 @@
import fcntl
import errno
import re
+import logging
import gevent
from gevent import socket
from gevent.queue import Queue
+END_OF_LINE = re.compile(r'\r\n|\n|\r')
+
class LineParser(object):
def __init__(self, handler, maxLineLength=None):
@@ -24,7 +27,7 @@ def __init__(self, handler, maxLineLength=None):
def write(self, text):
self._buf += text
while 1:
- m = re.search(r'\r\n|\n|\r', self._buf)
+ m = END_OF_LINE.search(self._buf)
ind = m.end() if m else None
if (ind == None
and self._maxLineLength is not None
@@ -46,12 +49,16 @@ def flush(self):
END_OF_FILE = ('__EOF__',)
-def safeRead(fd, chunkSize):
+def safeRead(fd, chunkSize, label):
try:
chunk = os.read(fd, 1024)
except OSError, ex:
if ex[0] == errno.EAGAIN:
return ''
+ if ex[0] == errno.EIO:
+ # what we see if the other end of the pipe dies uncleanly
+ return END_OF_FILE
+ logging.warning('safeRead exception, label=%s', label)
raise
if not chunk:
return END_OF_FILE
@@ -64,7 +71,7 @@ def setNonBlocking(fd):
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
-def copyFileToQueue(f, q, maxLineLength=None):
+def copyFileToQueue(f, q, maxLineLength=None, label=None):
"""
Given a file or file descriptor *f* (probably the output pipe from a
subprocess), asynchronously copy lines from the file into gevent
@@ -81,7 +88,7 @@ def copyFileToQueue(f, q, maxLineLength=None):
setNonBlocking(fd)
while 1:
- chunk = safeRead(fd, 1024)
+ chunk = safeRead(fd, 1024, label)
if chunk is END_OF_FILE:
break
if chunk:
@@ -102,7 +109,7 @@ def copyFileToQueue(f, q, maxLineLength=None):
q.put(StopIteration)
-def queueFromFile(f, maxLineLength=None):
+def queueFromFile(f, maxLineLength=None, label=None):
q = Queue()
- gevent.spawn(copyFileToQueue, f, q, maxLineLength)
+ gevent.spawn(copyFileToQueue, f, q, maxLineLength, label)
return q
@@ -8,37 +8,7 @@
from django.core.exceptions import ValidationError
from geocamUtil import anyjson as json
-
-
-def convertToDotDictRecurse(struct):
- if isinstance(struct, dict):
- for k, v in struct.iteritems():
- struct[k] = convertToDotDictRecurse(v)
- return DotDict(struct)
- elif isinstance(struct, list):
- return [convertToDotDictRecurse(elt) for elt in struct]
- else:
- return struct
-
-
-class DotDict(dict):
- # At the moment this object exists pretty much solely to let you
- # get and set elements in its __dict__ dictionary via dotted
- # notation. Someday it could do more.
-
- # these are fields that must not be defined to avoid causing problems
- # with Django
- _badFields = ('prepare_database_save',)
-
- def __repr__(self):
- return json.dumps(self, sort_keys=True, indent=4)
-
- def __getattr__(self, attr):
- if attr in self._badFields:
- raise KeyError(attr)
- return self.get(attr, None)
- __setattr__ = dict.__setitem__
- __delattr__ = dict.__delitem__
+from geocamUtil.dotDict import DotDict, convertToDotDictRecurse
class ExtrasDotField(models.TextField):
View
@@ -4,6 +4,8 @@
# All Rights Reserved.
# __END_LICENSE__
+import tempfile
+import os
class PdfConversionError(Exception):
pass
@@ -18,4 +20,26 @@ def convertPdf(data,
Pass in the raw binary PDF data. Returns the raw binary image data
result after rasterization.
"""
- raise NotImplementedError()
+
+ #write to temporary pdf file
+ tempInputFile = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False)
+ tempInputFile.seek(0)
+ tempInputFile.write(data)
+ tempInputFile.flush()
+
+ outputFileName = tempInputFile.name.replace('.pdf', '.png')
+
+ ret = os.system('convert -flatten %s %s > /dev/null' % (tempInputFile.name, outputFileName))
+
+ os.remove(tempInputFile.name)
+
+ if ret != 0 or os.path.isfile(outputFileName) == False:
+ raise PdfConversionError('Error found while converting pdf')
+
+ outputFile = open(outputFileName, 'r')
+ outputFileData = outputFile.read()
+
+ outputFile.close()
+ os.remove(outputFileName)
+
+ return outputFileData
View
@@ -16,6 +16,8 @@
import logging
import json
+loggerG = logging.getLogger('geocamUtil.store')
+
def encodeVal(val):
return zlib.compress(pickle.dumps(val, pickle.HIGHEST_PROTOCOL), 9)
@@ -158,6 +160,7 @@ def purgeQueue(self):
cacheInfo.refCount -= 1
def flushEntry(self, key):
+ loggerG.debug('flushEntry %s', key)
cacheInfo = self.cacheInfo[key]
if cacheInfo.dirty:
val = self.cache[key]
@@ -167,28 +170,32 @@ def flushEntry(self, key):
cacheInfo.dirty = False
def evictEntry(self, key):
+ loggerG.debug('evictEntry %s', key)
self.flushEntry(key)
del self.cache[key]
del self.cacheInfo[key]
def evictLru(self):
while len(self.cache) >= self.maxEntries:
key = self.queue.popleft()
- cacheInfo = self.cacheInfo[key]
- if cacheInfo.refCount == 1:
- self.evictEntry(key)
- else:
- cacheInfo.refCount -= 1
+ cacheInfo = self.cacheInfo.get(key, None)
+ if cacheInfo:
+ if cacheInfo.refCount == 1:
+ self.evictEntry(key)
+ else:
+ cacheInfo.refCount -= 1
def sync(self):
- for key in self.cache.keys():
- self.flushEntry(key)
+ for key, cacheInfo in self.cacheInfo.iteritems():
+ if cacheInfo.dirty:
+ self.flushEntry(key)
def __getitem__(self, key):
if key in self.cache:
self.markRead(key)
return self.cache[key]
else:
+ # hm... add to cache?
return self.store[key]
def __setitem__(self, key, val):
@@ -7,6 +7,7 @@
import shutil
import unittest
import tempfile
+import logging
from geocamUtil.store import FileStore, LruCacheStore
@@ -16,18 +17,26 @@ def storeTest(self, storeFactory):
tempDir = tempfile.mkdtemp('-storeTestDir')
store1 = storeFactory(tempDir)
- store1['a'] = 1
- store1['b'] = 2
+ for i in xrange(20):
+ store1[str(i)] = i
store1.sync()
store2 = storeFactory(tempDir)
- self.assertEqual(store2['a'], 1)
- self.assertEqual(store2['b'], 2)
+ for i in xrange(20):
+ self.assertEqual(store2[str(i)], i)
shutil.rmtree(tempDir)
def test_FileStore(self):
self.storeTest(FileStore)
def test_LruCacheStore(self):
- self.storeTest(lambda path: LruCacheStore(FileStore(path), 100))
+ self.storeTest(lambda path: LruCacheStore(FileStore(path), 5))
+
+# def test_LruEvict(self):
+# pass
+
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.DEBUG)
+ unittest.main()
@@ -12,6 +12,10 @@
from IPython.config.loader import Config
from IPython.frontend.terminal.embed import InteractiveShellEmbed
from IPython.lib.inputhook import inputhook_manager, stdin_ready
+try:
+ from django.conf import settings
+except ImportError:
+ settings = object()
from geocamUtil.jsonConfig import loadConfig
from geocamUtil.zmq.zerorpcClientProxy import ClientProxy
@@ -88,7 +92,7 @@ def main():
import optparse
parser = optparse.OptionParser('usage: %prog OPTIONS')
parser.add_option('-p', '--ports',
- default='ports.json',
+ default=getattr(settings, 'GEOCAM_UTIL_ZMQ_PORTS_PATH', 'ports.json'),
help='Path to ports config file [%default]')
opts, args = parser.parse_args()
if args:
@@ -11,7 +11,7 @@ class %(decoratedName)s(%(parentName)s):
FUNC_TEMPLATE = """
def %(name)s(self, %(signature)s):
%(docstring)s
- return self._client.%(name)s(%(signature)s)
+ return self._client.%(name)s(%(argNames)s)
"""
@@ -27,11 +27,15 @@ class MissingVal(object):
MISSING = MissingVal()
-def declarationFromZerorpcInspectArg(arg):
+def nameFromArg(arg):
name = arg['name']
if isinstance(name, tuple):
name = '(%s)' % ', '.join([n for n in name])
- result = name
+ return name
+
+
+def declarationFromZerorpcInspectArg(arg):
+ result = nameFromArg(arg)
defaultVal = arg.get('default', MISSING)
if defaultVal is not MISSING:
@@ -66,13 +70,16 @@ def makeDecoratedProxy(cls, name, client):
methods = meta['methods']
for name, info in methods.iteritems():
- argNames = [declarationFromZerorpcInspectArg(arg) for arg in info['args']]
- signature = ', '.join(argNames[1:])
+ argSigs = [declarationFromZerorpcInspectArg(arg) for arg in info['args']]
+ signature = ', '.join(argSigs[1:])
+ argNames = ', '.join(nameFromArg(arg) for arg in info['args'][1:])
docstring = repr(info.get('doc', '<unknown>'))
src += (FUNC_TEMPLATE
% {'name': name,
'signature': signature,
+ 'argNames': argNames,
'docstring': docstring})
+ # print 'zerorpcClientProxy:', src
code = compile(src, '<string>', 'single')
evaldict = {'__name__': 'geocamUtil.zmq.zerorpcClientProxy',
'ClientProxy': ClientProxy}
Oops, something went wrong.

0 comments on commit 48c5416

Please sign in to comment.