Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Added file_pool implementation and tests

  • Loading branch information...
commit d132a6d45d5c1343af2a9245d7859b7b0cddd614 1 parent 3e760ce
@nibrahim nibrahim authored
View
108 liveweb/file_pool.py
@@ -0,0 +1,108 @@
+"""
+File pool implementation
+"""
+
+import datetime
+import os
+import Queue
+import random
+import threading
+
+import logging
+logging.basicConfig(level = logging.DEBUG)
+
+class MemberFile(object):
+ """
+ """
+ def __init__(self, name, pool, *largs, **kargs):
+ self.fp = open(name, *largs, **kargs)
+ self.pool = pool
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.pool.return_file(self)
+
+
+ def __getattr__(self, attr):
+ return getattr(self.fp, attr)
+
+
+
+class FilePool(object):
+ """
+ Implements a pool of files from which a file can be requested.
+
+ """
+ def __init__(self, directory, pattern, max_files, max_file_size):
@anandology Owner

pattern, max_files, max_file_size should have reasonable defaults.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ """
+ Creates a pool of files in the given directory with the
+ specified pattern.
+
+ The number of files is max_files and the maximum size of each
+ file is max_file_size.
+
+ The `get_file` method returns a new file from the pool
+
+ """
+ self.directory = directory
+ self.pattern = pattern
+ self.max_files = max_files
+ self.max_file_size = max_file_size
+
+ self.queue = Queue.Queue(self.max_files)
+
+ self.seq = 0
@anandology Owner

Whenever we restart the app, the seq is going to reset. Shouldn't we continue the existing sequence by looking at the files on disk?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+
+ for i in range(self.max_files):
+ self._add_file_to_pool()
+
+ def _add_file_to_pool(self):
+ "Creates a new file and puts it in the pool"
+ pattern_dict = dict(timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%s%f"),
+ seq = "%05d"%self.seq)
@anandology Owner

why %05d? Shouldn't that be decided by the pattern?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+ fname = self.pattern%pattern_dict
+ absolute_name = os.path.join(self.directory, fname)
+ logging.debug("Adding %s to pool",absolute_name)
+ fp = MemberFile(absolute_name, self, mode = "ab")
+ self.queue.put_nowait(fp)
+ self.seq += 1
@anandology Owner

This is not thread-safe.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
+
+ def return_file(self, f):
+ """Returns a file to the pool. Will discard the file and
+ insert a new one if the file is above max_file_size."""
+ logging.debug("Returning %s",f)
+ file_size = f.tell()
+ if file_size < self.max_file_size:
+ logging.debug(" Put it back")
+ self.queue.put(f)
+ else:
+ logging.debug(" Closing and creating a new file")
+ f.close()
+ self._add_file_to_pool()
+
+ def get_file(self):
+ f = self.queue.get()
+ logging.debug("Getting %s",f)
+ return f
+
+ def close(self):
+ logging.debug("Closing all descriptors. Emptying pool.")
+ while not self.queue.empty():
+ fp = self.queue.get_nowait()
+ fp.close()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
View
17 liveweb/tests/conftest.py
@@ -0,0 +1,17 @@
+import os
+import shutil
+
+def pytest_funcarg__pooldir(request):
+ "Creates a directory for the pool"
+ dirname = "/tmp/pool-xxx"
+
+ if os.path.exists(dirname):
+ shutil.rmtree(dirname)
+
+ os.mkdir(dirname)
+
+ request.addfinalizer(lambda : shutil.rmtree(dirname))
+
+ return dirname
+
+
View
121 liveweb/tests/test_filepool.py
@@ -0,0 +1,121 @@
+import glob
+import os
+
+def test_creation(pooldir):
+ """
+ Tests to see if the file pool is created properly.
+
+ Creates a pool and then walks through the pool directory to see if
+ the expected files are there.
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ # Get files in pool directory.
+ pool_files = set(glob.glob(pooldir + "/*"))
+
+ # Check if this is the same as what we expect
+ expected_files = set(["%s/test-%05d"%(pooldir,x) for x in range(0,10)])
+
+ assert expected_files == pool_files
+
+def test_get_return(pooldir):
+ """
+ Tests to see if get_file return_file work as expected.
+
+ gets files, checks the number of files in the queue, returns them
+ and checks again.
+ """
+
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ fps = []
+
+ assert len(pool.queue.queue) == 10
+
+ for i in range(1, 6):
+ fps.append(pool.get_file())
+ assert len(pool.queue.queue) == 10 - i
+
+ assert len(pool.queue.queue) == 5
+
+ for i in range(1, 6):
+ pool.return_file(fps.pop())
+ assert len(pool.queue.queue) == 5 + i
+
+ assert len(pool.queue.queue) == 10
+
+def test_max_file_size(pooldir):
+ """
+ Tests to see if the files are closed and released when the maximum
+ file size is reached and the file is returned.
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ fp = pool.get_file()
+ fp.write("test" * 100) # Max size has been exceeded. File should
+ pool.return_file(fp) # get removed from pool when returned.
+
+ pool_files = set(x.fp.name for x in pool.queue.queue)
+ expected_files = set(["%s/test-%05d"%(pooldir,x) for x in range(1,11)])
+
+ assert expected_files == pool_files
+
+
+def test_close_pool(pooldir):
+ """
+ Makes sure that the pool is emptied when closed.
+
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ pool.close()
+
+ assert len(pool.queue.queue) == 0
+
+
+def test_member_file_context(pooldir):
+ """
+ Tests the context manager behaviour of the MemberFile object.
+ """
+
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ assert len(pool.queue.queue) == 10
+
+ with pool.get_file() as f:
+ assert len(pool.queue.queue) == 9
+ name = f.name
+ f.write("Hello")
+
+ assert len(pool.queue.queue) == 10
+ pool.close()
+
+ assert open(name).read() == "Hello"
+
+
+
+
+
+
+
+
+
+
+
+
+
Please sign in to comment.
Something went wrong with that request. Please try again.