Skip to content

Commit

Permalink
Pylint cleanups.
Browse files Browse the repository at this point in the history
  • Loading branch information
@koldinger
koldinger committed Dec 15, 2016
1 parent 7888c57 commit 2d28cc0
Show file tree
Hide file tree
Showing 22 changed files with 521 additions and 661 deletions.
12 changes: 6 additions & 6 deletions Tardis/Cache.py
View file
Expand Up @@ -53,10 +53,10 @@ def insert(self, key, value, now=None, timeout=None):
self.cache[key] = (value, timeout)
self.logger.debug("Inserting key %s", key)
if self.size != 0 and len(self.cache) > self.size:
self.cache.flush()
self.flush()
if len(self.cache) > self.size:
self.cache.popitem(False)

def retrieve(self, key):
if not key in self.cache:
self.logger.debug("Retrieving key %s failed", key)
Expand All @@ -73,15 +73,15 @@ def retrieve(self, key):
def delete(self, key):
if key in self.cache:
del self.cache[key]

def flush(self):
now = time.time()
i = self.cache.iteritems()
z = i.next()
try:
while z:
(key, item) = z
(value, timeout) = item
(_, item) = z
(_, timeout) = item
if timeout > now:
return
self.cache.popitem(False)
Expand All @@ -91,7 +91,7 @@ def flush(self):
pass

def purge(self):
self.cache = collection.OrderedDict()
self.cache = collections.OrderedDict()

if __name__ == "__main__":
c = Cache(5, 2)
Expand Down
23 changes: 10 additions & 13 deletions Tardis/CacheDir.py
View file
Expand Up @@ -32,10 +32,9 @@
import os.path
import socket
import logging
import pwd, grp
import shutil

class CacheDir:
class CacheDir(object):
def __init__(self, root, parts=2, partsize=2, create=True, user=None, group=None):
self.root = os.path.abspath(root)
self.parts = parts
Expand All @@ -48,18 +47,18 @@ def __init__(self, root, parts=2, partsize=2, create=True, user=None, group=None
if create:
os.makedirs(self.root)
if self.chown:
os.chown(self.root, self.user, self.group)
os.chown(self.root, self.user, self.group)
else:
raise Exception("CacheDir does not exist: " + root)

def comps(self, name):
return [name[(i * self.partsize):((i + 1) * self.partsize)] for i in range(0, self.parts)]

def dir(self, name):
def dirPath(self, name):
return reduce(os.path.join, self.comps(name), self.root)

def path(self, name):
return os.path.join(self.dir(name), name)
return os.path.join(self.dirPath(name), name)

def exists(self, name):
return os.path.lexists(self.path(name))
Expand All @@ -72,9 +71,9 @@ def size(self, name):
return 0

def mkdir(self, name):
dir = self.dir(name)
if not os.path.isdir(dir):
os.makedirs(dir)
directory = self.dirPath(name)
if not os.path.isdir(directory):
os.makedirs(directory)
if self.chown:
path = self.root
for i in self.comps(name):
Expand All @@ -100,7 +99,7 @@ def insert(self, name, source):

def link(self, source, dest):
dstpath = self.path(dest)
srcpath = os.path.relpath(self.path(source), self.dir(dest))
srcpath = os.path.relpath(self.path(source), self.dirPath(dest))
os.symlink(srcpath, dstpath)

def remove(self, name):
Expand All @@ -117,7 +116,7 @@ def removeSuffixes(self, name, suffixes):
#logger.debug("Removed %s", name + suffix)
deleted += 1
return deleted

def move(self, oldname, newname):
try:
os.rename(self.path(oldname), self.path(newname))
Expand All @@ -132,7 +131,7 @@ def move(self, oldname, newname):
path = os.path.join("cache", socket.gethostname())
c = CacheDir(path, 4, 2, True)
print c.comps(test)
print c.dir(test)
print c.dirPath(test)
print c.path(test)
print c.exists(test)

Expand All @@ -149,5 +148,3 @@ def move(self, oldname, newname):
for line in fd:
print line,
print c.exists(test)


64 changes: 24 additions & 40 deletions Tardis/CompressedBuffer.py
View file
Expand Up @@ -29,11 +29,9 @@
# POSSIBILITY OF SUCH DAMAGE.

import zlib
import base64
import sys
import StringIO
import hashlib
import librsync

import Tardis.librsync as librsync

_defaultChunksize = 128 * 1024

Expand Down Expand Up @@ -66,7 +64,7 @@ def read(self, size=0x7fffffff):
#print "read called: {} {} bytes available".format(size, avail)
out = ""
left = size
while(len(out) < size):
while len(out) < size:
#print "read loop: so far: {}".format(len(out))
if (not self.buffer) or (len(self.buffer) == 0):
#print "Calling _get"
Expand Down Expand Up @@ -114,37 +112,36 @@ def _get(self):
ret = None
if self.stream:
while not ret:
buffer = self.stream.read(self.chunksize)
self.uncompressed += len(buffer)
buf = self.stream.read(self.chunksize)
self.uncompressed += len(buf)
if self.hasher:
self.hasher.update(buffer)
self.hasher.update(buf)
if self.sig:
self.sig.step(buffer)
self.sig.step(buf)
# First time around, create a compressor and check the compression ratio
if self.first:
self.first = False
buflen = len(buffer)
self.compressor = zlib.compressobj()
ret = self.compressor.compress(buffer)
# Flush the buffer and colculate the size
ret = self.compressor.compress(buf)
# Flush the buf and colculate the size
ret += self.compressor.flush(zlib.Z_SYNC_FLUSH)
# Now, check what we've got back.
if ret:
ratio = float(len(ret)) / float(len(buffer))
#print "Initial ratio: {} {} {}".format(ratio, len(ret), len(buffer))
ratio = float(len(ret)) / float(len(buf))
#print "Initial ratio: {} {} {}".format(ratio, len(ret), len(buf))
if ratio > self.threshold:
ret = buffer
ret = buf
self.compressor = None
elif self.compressor:
if not buffer:
if not buf:
#print "_get: Done"
ret = self.compressor.flush(zlib.Z_FINISH)
self.stream = None
else:
#print "_get: {} bytes read".format(len(buffer))
ret = self.compressor.compress(buffer)
#print "_get: {} bytes read".format(len(buf))
ret = self.compressor.compress(buf)
else:
ret = buffer
ret = buf
break # Make sure we don't got around the loop at the EOF

self.compressed += len(ret)
Expand All @@ -158,11 +155,11 @@ def compsize(self):
return self.compressed

def ratio(self):
return (float(self.compressed) / float(self.uncompressed))
return float(self.compressed) / float(self.uncompressed)

def size(self):
return self.origsize()

def isCompressed(self):
return self.compressor != None

Expand All @@ -178,23 +175,23 @@ def _get(self):
ret = None
while not ret:
if self.stream:
buffer = self.stream.read(self.chunksize)
if not buffer:
buf = self.stream.read(self.chunksize)
if not buf:
#print "_get: Done"
ret = self.compressor.flush()
self.uncompressed = self.uncompressed + len(ret)
self.stream = None
else:
#print "_get: {} bytes read".format(len(buffer))
ret = self.compressor.decompress(buffer)
self.compressed = self.uncompressed + len(buffer)
#print "_get: {} bytes read".format(len(buf))
ret = self.compressor.decompress(buf)
self.compressed = self.uncompressed + len(buf)
self.uncompressed = self.compressed + len(ret)
return ret
return None

if __name__ == "__main__":
print "Opening {}".format(sys.argv[1])
x = CompressedBufferedReader(file(sys.argv[1], "rb"), checksum=True)
x = CompressedBufferedReader(file(sys.argv[1], "rb"))
#line = x.get()
with file(sys.argv[2], "wb") as f:
line = x.read(16384)
Expand All @@ -205,16 +202,3 @@ def _get(self):
line = x.read(16384)

print x.origsize(), " ", x.compsize(), " ", x.ratio(), " :: ", x.checksum()

"""
print "Opening {}".format(sys.argv[2])
y = UncompressedBufferedReader(file(sys.argv[2], "rb"))
total = 0
line = y.read(size=80)
while line:
#print "==== ", len(line), ":", total, " :: ", line #base64.b64encode(line)
#print line
total += len(line)
#line = x.get()
line = y.read(size=80)
"""
15 changes: 8 additions & 7 deletions Tardis/Config.py
View file
Expand Up @@ -28,17 +28,18 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import ConfigParser
import Defaults
import argparse
import Util
import sys
import ConfigParser

import Tardis.Defaults as Defaults
import Tardis.Util as Util


configDefaults = {
'Database': Defaults.getDefault('TARDIS_DB'),
'Client': Defaults.getDefault('TARDIS_CLIENT'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'DBDir': Defaults.getDefault('TARDIS_DBDIR'),
'DBName': Defaults.getDefault('TARDIS_DBNAME'),
'Password': None,
'PasswordFile': None,
'PasswordProg': None,
Expand Down Expand Up @@ -86,5 +87,5 @@ def addPasswordOptions(parser):
pwgroup.add_argument('--password-prog', dest='passwordprog', default=config.get(job, 'PasswordProg'), help='Use the specified command to generate the password on stdout')

passgroup.add_argument('--crypt', dest='crypt',action=Util.StoreBoolean, default=config.getboolean(job, 'Crypt'),
help='Encrypt data. Only valid if password is set')
help='Encrypt data. Only valid if password is set')
passgroup.add_argument('--keys', dest='keys', default=config.get(job, 'KeyFile'), help='Load keys from file.')
28 changes: 10 additions & 18 deletions Tardis/Connection.py
View file
Expand Up @@ -31,11 +31,11 @@
import socket
import json
import uuid
import sys
import time
import Messages
import ssl

import Tardis
import Tardis.Messages as Messages

protocolVersion = "1.1"
headerString = "TARDIS " + protocolVersion
Expand Down Expand Up @@ -80,7 +80,7 @@ def __init__(self, host, port, name, encoding, priority, client, autoname, token
raise Exception("Unknown protocol: {}".format(message))

# Create a BACKUP message
message = {
resp = {
'message' : 'BACKUP',
'host' : client,
'encoding' : encoding,
Expand All @@ -94,9 +94,9 @@ def __init__(self, host, port, name, encoding, priority, client, autoname, token
'full' : full
}
if token:
message['token'] = token
resp['token'] = token
# BACKUP { json message }
self.put(json.dumps(message))
self.put(json.dumps(resp))

message = self.sock.recv(1024).strip()
fields = json.loads(message)
Expand All @@ -114,7 +114,7 @@ def __init__(self, host, port, name, encoding, priority, client, autoname, token
self.filenameKey = fields['filenameKey']
if 'contentKey' in fields:
self.contentKey = fields['contentKey']
except Exception as e:
except Exception:
self.sock.close()
raise

Expand All @@ -123,7 +123,7 @@ def put(self, message):
self.stats['messagesSent'] += 1
return

def recv(n):
def recv(self, n):
msg = ''
while len(msg) < n:
chunk = self.sock.recv(n-len(msg))
Expand Down Expand Up @@ -206,18 +206,10 @@ def __init__(self, host, port, name, priority=0, client=None, autoname=False, to
# Really, cons this up in the connection, but it needs access to the sock parameter, so.....
self.sender = Messages.MsgPackMessages(self.sock, stats=self.stats, compress=compress)

class NullConnection(Connection):
def __init__(self, host, port, name):
pass

def send(self, message):
print json.dumps(message)

def receive(self):
return None

if __name__ == "__main__":
""" Test Code """
"""
Test Code
"""
conn = JsonConnection("localhost", 9999, "HiMom")
print conn.getSessionId()
conn.send({ 'x' : 1 })
Expand Down

0 comments on commit 2d28cc0

Please sign in to comment.