Permalink
Browse files

Added file_pool implementation and tests

  • Loading branch information...
1 parent 3e760ce commit d132a6d45d5c1343af2a9245d7859b7b0cddd614 @nibrahim nibrahim committed Apr 20, 2012
Showing with 246 additions and 0 deletions.
  1. +108 −0 liveweb/file_pool.py
  2. +17 −0 liveweb/tests/conftest.py
  3. +121 −0 liveweb/tests/test_filepool.py
View
@@ -0,0 +1,108 @@
+"""
+File pool implementation
+"""
+
+import datetime
+import os
+import Queue
+import random
+import threading
+
+import logging
+logging.basicConfig(level = logging.DEBUG)
+
+class MemberFile(object):
+ """
+ """
+ def __init__(self, name, pool, *largs, **kargs):
+ self.fp = open(name, *largs, **kargs)
+ self.pool = pool
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.pool.return_file(self)
+
+
+ def __getattr__(self, attr):
+ return getattr(self.fp, attr)
+
+
+
+class FilePool(object):
+ """
+ Implements a pool of files from which a file can be requested.
+
+ """
+ def __init__(self, directory, pattern, max_files, max_file_size):
@anandology

anandology Apr 20, 2012

Collaborator

pattern, max_files, max_file_size should have reasonable defaults.

+ """
+ Creates a pool of files in the given directory with the
+ specified pattern.
+
+ The number of files is max_files and the maximum size of each
+ file is max_file_size.
+
+ The `get_file` method returns a new file from the pool
+
+ """
+ self.directory = directory
+ self.pattern = pattern
+ self.max_files = max_files
+ self.max_file_size = max_file_size
+
+ self.queue = Queue.Queue(self.max_files)
+
+ self.seq = 0
@anandology

anandology Apr 20, 2012

Collaborator

Whenever we restart the app, the seq is going to reset. Shouldn't we continue the existing sequence by looking at the files on disk?

+
+ for i in range(self.max_files):
+ self._add_file_to_pool()
+
+ def _add_file_to_pool(self):
+ "Creates a new file and puts it in the pool"
+ pattern_dict = dict(timestamp = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%s%f"),
+ seq = "%05d"%self.seq)
@anandology

anandology Apr 20, 2012

Collaborator

why %05d? Shouldn't that be decided by the pattern?

+ fname = self.pattern%pattern_dict
+ absolute_name = os.path.join(self.directory, fname)
+ logging.debug("Adding %s to pool",absolute_name)
+ fp = MemberFile(absolute_name, self, mode = "ab")
+ self.queue.put_nowait(fp)
+ self.seq += 1
@anandology

anandology Apr 20, 2012

Collaborator

This is not thread-safe.

+
+ def return_file(self, f):
+ """Returns a file to the pool. Will discard the file and
+ insert a new one if the file is above max_file_size."""
+ logging.debug("Returning %s",f)
+ file_size = f.tell()
+ if file_size < self.max_file_size:
+ logging.debug(" Put it back")
+ self.queue.put(f)
+ else:
+ logging.debug(" Closing and creating a new file")
+ f.close()
+ self._add_file_to_pool()
+
+ def get_file(self):
+ f = self.queue.get()
+ logging.debug("Getting %s",f)
+ return f
+
+ def close(self):
+ logging.debug("Closing all descriptors. Emptying pool.")
+ while not self.queue.empty():
+ fp = self.queue.get_nowait()
+ fp.close()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
View
@@ -0,0 +1,17 @@
+import os
+import shutil
+
+def pytest_funcarg__pooldir(request):
+ "Creates a directory for the pool"
+ dirname = "/tmp/pool-xxx"
+
+ if os.path.exists(dirname):
+ shutil.rmtree(dirname)
+
+ os.mkdir(dirname)
+
+ request.addfinalizer(lambda : shutil.rmtree(dirname))
+
+ return dirname
+
+
@@ -0,0 +1,121 @@
+import glob
+import os
+
+def test_creation(pooldir):
+ """
+ Tests to see if the file pool is created properly.
+
+ Creates a pool and then walks through the pool directory to see if
+ the expected files are there.
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ # Get files in pool directory.
+ pool_files = set(glob.glob(pooldir + "/*"))
+
+ # Check if this is the same as what we expect
+ expected_files = set(["%s/test-%05d"%(pooldir,x) for x in range(0,10)])
+
+ assert expected_files == pool_files
+
+def test_get_return(pooldir):
+ """
+ Tests to see if get_file return_file work as expected.
+
+ gets files, checks the number of files in the queue, returns them
+ and checks again.
+ """
+
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ fps = []
+
+ assert len(pool.queue.queue) == 10
+
+ for i in range(1, 6):
+ fps.append(pool.get_file())
+ assert len(pool.queue.queue) == 10 - i
+
+ assert len(pool.queue.queue) == 5
+
+ for i in range(1, 6):
+ pool.return_file(fps.pop())
+ assert len(pool.queue.queue) == 5 + i
+
+ assert len(pool.queue.queue) == 10
+
+def test_max_file_size(pooldir):
+ """
+ Tests to see if the files are closed and released when the maximum
+ file size is reached and the file is returned.
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ fp = pool.get_file()
+ fp.write("test" * 100) # Max size has been exceeded. File should
+ pool.return_file(fp) # get removed from pool when returned.
+
+ pool_files = set(x.fp.name for x in pool.queue.queue)
+ expected_files = set(["%s/test-%05d"%(pooldir,x) for x in range(1,11)])
+
+ assert expected_files == pool_files
+
+
+def test_close_pool(pooldir):
+ """
+ Makes sure that the pool is emptied when closed.
+
+ """
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ pool.close()
+
+ assert len(pool.queue.queue) == 0
+
+
+def test_member_file_context(pooldir):
+ """
+ Tests the context manager behaviour of the MemberFile object.
+ """
+
+ from ..file_pool import FilePool
+
+ # Create the pool
+ pool = FilePool(pooldir, pattern = "test-%(seq)s", max_files = 10, max_file_size = 10)
+
+ assert len(pool.queue.queue) == 10
+
+ with pool.get_file() as f:
+ assert len(pool.queue.queue) == 9
+ name = f.name
+ f.write("Hello")
+
+ assert len(pool.queue.queue) == 10
+ pool.close()
+
+ assert open(name).read() == "Hello"
+
+
+
+
+
+
+
+
+
+
+
+
+

0 comments on commit d132a6d

Please sign in to comment.