Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Moving items around

  • Loading branch information...
commit 59da5764fce1d33f1639fb16dcdc5e7f3bf2cab9 1 parent 4ff565b
Kevin Landreth CrackerJackMack authored

Showing 1 changed file with 87 additions and 87 deletions. Show diff stats Hide diff stats

  1. +87 87 slbackup.py
174 slbackup.py
@@ -194,6 +194,93 @@ def authenticate(self):
194 194 del client
195 195
196 196
  197 +def chunk_upload(obj, filename, headers=None):
  198 + upload = obj.chunk_upload(headers=headers)
  199 + with open(filename, 'rb') as _f:
  200 + for line in asblocks(_f):
  201 + upload.send(line)
  202 + upload.finish()
  203 +
  204 +
  205 +def get_filesize(_f):
  206 + if isinstance(_f, file):
  207 + size = int(os.fstat(_f.fileno())[6])
  208 + else:
  209 + with open(_f) as data:
  210 + size = int(os.fstat(data.fileno())[6])
  211 +
  212 + return size
  213 +
  214 +
  215 +def swifthash(_f):
  216 + """ Compute md5 of the file for comparison """
  217 +
  218 + m = md5()
  219 + with open(_f, 'rb') as data:
  220 + for line in asblocks(data):
  221 + m.update(line)
  222 +
  223 + return m.hexdigest()
  224 +
  225 +
  226 +def asblocks(_f, buflen=_DEFAULT_OS_BUFLEN):
  227 + """Generator that yields buflen bytes from an open filehandle.
  228 + Yielded bytes might be less buflen. """
  229 + if not isinstance(_f, file):
  230 + raise TypeError("First parameter must be an file object")
  231 +
  232 + try:
  233 + while True:
  234 + data = _f.read(buflen)
  235 + if data:
  236 + yield data
  237 + else:
  238 + break
  239 + except IOError, e:
  240 + logging.error("Failed to read %d bytes: %s", buflen, e)
  241 + raise e
  242 +
  243 +
  244 +def queue_iter(queue):
  245 + while True:
  246 + try:
  247 + item = queue.get()
  248 + except Queue.Empty:
  249 + break
  250 +
  251 + if item is None:
  252 + break
  253 +
  254 + yield item
  255 +
  256 +
  257 +def roundrobin_iter(**queues):
  258 + total_queues = len(queues)
  259 + miss = 0
  260 + for name, q in repeat(queues.iteritems()):
  261 + try:
  262 + item = q.get(False)
  263 + except (Queue.Empty, TimeoutError):
  264 + miss += 1
  265 + if miss > total_queues:
  266 + break
  267 + continue
  268 + else:
  269 + miss = 0
  270 + yield name, item
  271 +
  272 +
  273 +class IterUnwrap(object):
  274 + def __init__(self, func, *args, **kwargs):
  275 + self.func = func
  276 + self.args = args
  277 + self.kwargs = kwargs
  278 +
  279 + def __call__(self, item):
  280 + a = (item,) + self.args
  281 + return self.func(*a, **self.kwargs)
  282 +
  283 +
197 284 def get_container(app, name=None):
198 285 if name is None:
199 286 name = app.container
@@ -650,93 +737,6 @@ def upload_file(job, app, jobs):
650 737 container = get_container(app)
651 738
652 739
653   -def chunk_upload(obj, filename, headers=None):
654   - upload = obj.chunk_upload(headers=headers)
655   - with open(filename, 'rb') as _f:
656   - for line in asblocks(_f):
657   - upload.send(line)
658   - upload.finish()
659   -
660   -
661   -def get_filesize(_f):
662   - if isinstance(_f, file):
663   - size = int(os.fstat(_f.fileno())[6])
664   - else:
665   - with open(_f) as data:
666   - size = int(os.fstat(data.fileno())[6])
667   -
668   - return size
669   -
670   -
671   -def swifthash(_f):
672   - """ Compute md5 of the file for comparison """
673   -
674   - m = md5()
675   - with open(_f, 'rb') as data:
676   - for line in asblocks(data):
677   - m.update(line)
678   -
679   - return m.hexdigest()
680   -
681   -
682   -def asblocks(_f, buflen=_DEFAULT_OS_BUFLEN):
683   - """Generator that yields buflen bytes from an open filehandle.
684   - Yielded bytes might be less buflen. """
685   - if not isinstance(_f, file):
686   - raise TypeError("First parameter must be an file object")
687   -
688   - try:
689   - while True:
690   - data = _f.read(buflen)
691   - if data:
692   - yield data
693   - else:
694   - break
695   - except IOError, e:
696   - logging.error("Failed to read %d bytes: %s", buflen, e)
697   - raise e
698   -
699   -
700   -def queue_iter(queue):
701   - while True:
702   - try:
703   - item = queue.get()
704   - except Queue.Empty:
705   - break
706   -
707   - if item is None:
708   - break
709   -
710   - yield item
711   -
712   -
713   -def roundrobin_iter(**queues):
714   - total_queues = len(queues)
715   - miss = 0
716   - for name, q in repeat(queues.iteritems()):
717   - try:
718   - item = q.get(False)
719   - except (Queue.Empty, TimeoutError):
720   - miss += 1
721   - if miss > total_queues:
722   - break
723   - continue
724   - else:
725   - miss = 0
726   - yield name, item
727   -
728   -
729   -class IterUnwrap(object):
730   - def __init__(self, func, *args, **kwargs):
731   - self.func = func
732   - self.args = args
733   - self.kwargs = kwargs
734   -
735   - def __call__(self, item):
736   - a = (item,) + self.args
737   - return self.func(*a, **self.kwargs)
738   -
739   -
740 740 if __name__ == "__main__":
741 741 import optparse
742 742

0 comments on commit 59da576

Please sign in to comment.
Something went wrong with that request. Please try again.