Permalink
Browse files

a new hardline/copy fallback mechanism

It's more testable
  • Loading branch information...
1 parent 7b0c392 commit b755f3074d2685ad3773f0bc16697d38faea8100 @jinty jinty committed Jan 13, 2012
Showing with 61 additions and 20 deletions.
  1. +13 −8 van/static/cdn.py
  2. +48 −12 van/static/tests/test_cdn.py
View
@@ -145,6 +145,8 @@ def _walk_resources(resources):
class _PutLocal:
+ _hard_link = True
+
def __init__(self, target):
assert target.startswith('file:///')
self._target_dir = target = target[7:]
@@ -177,16 +179,19 @@ def put(self, files):
self._if_not_exist(os.makedirs, target)
def _copy(self, source, target):
- # If source and target is on the same device, use shutil.copy
- if os.stat(os.path.dirname(target)).st_dev == os.stat(source).st_dev:
- logging.debug("Hard linking %s to %s", source, target)
- copy_function = os.link
- # Otherwise use os.link and create a hard link
+ if self._hard_link:
+ try:
+ logging.debug("Hard linking %s to %s", source, target)
+ self._if_not_exist(os.link, source, target) # hard links are fast!
+ except:
+ logging.debug("Hard linking failed, falling back to normal copy")
+ # don't try hard linking after first failure
+ # this may be because the files are on differnt devices or windows
+ self._hard_link = False
+ self._copy(source, target)
else:
logging.debug("Copying %s to %s", source, target)
- copy_function = shutil.copy
-
- self._if_not_exist(copy_function, source, target)
+ self._if_not_exist(shutil.copy, source, target) # hard links are fast!
class _PutS3:
@@ -179,7 +179,7 @@ def test_walk(self):
])
-class TestPutLocal(TestCase):
+class TestPutLocalMixin:
def setUp(self):
import tempfile
@@ -189,34 +189,33 @@ def tearDown(self):
import shutil
shutil.rmtree(self._tmpdir)
+ def make_one(self):
+ target_url = 'file://%s' % self._tmpdir
+ from van.static.cdn import _PutLocal
+ return _PutLocal(target_url)
+
def test_put_twice(self):
# we can put to local twice without issue
# https://github.com/jinty/van.static/issues/1
here = os.path.dirname(__file__)
- target_url = 'file://%s' % self._tmpdir
from pkg_resources import get_distribution
dist = get_distribution('van.static')
- here = os.path.dirname(__file__)
- from van.static.cdn import _PutLocal
- putter = _PutLocal(target_url)
+ one = self.make_one()
to_put = [
('tests/example', here + '/example', 'van.static', dist, 'dir'),
('tests/example/css', here + '/example/css', 'van.static', dist, 'dir'),
('tests/example/css/example.css', here + '/example/css/example.css', 'van.static', dist, 'file'),
('tests/example/example.txt', here + '/example/example.txt', 'van.static', dist, 'file'),
]
- putter.put(to_put)
- putter.put(to_put)
+ one.put(to_put)
+ one.put(to_put)
def test_put(self):
here = os.path.dirname(__file__)
- target_url = 'file://%s' % self._tmpdir
from pkg_resources import get_distribution
dist = get_distribution('van.static')
- here = os.path.dirname(__file__)
- from van.static.cdn import _PutLocal
- putter = _PutLocal(target_url)
- putter.put([
+ one = self.make_one()
+ one.put([
('tests/example', here + '/example', 'van.static', dist, 'dir'),
('tests/example/css', here + '/example/css', 'van.static', dist, 'dir'),
('tests/example/css/example.css', here + '/example/css/example.css', 'van.static', dist, 'file'),
@@ -241,6 +240,43 @@ def test_put(self):
self.assertEqual(os.listdir(d), ['example.css'])
+class TestPutLocal(TestPutLocalMixin, TestCase):
+
+ @patch('os.link')
+ @patch('shutil.copy')
+ def test_fallback_to_copy(self, copy, link):
+ from pkg_resources import get_distribution
+ dist = get_distribution('van.static')
+ one = self.make_one()
+ to_put = [
+ ('tests/example/css/example.css', '/example/css/example.css', 'van.static', dist, 'file'),
+ ('tests/example/example.txt', '/example/example.txt', 'van.static', dist, 'file'),
+ ]
+ # if link never fails it gets called twice
+ self.assertTrue(one._hard_link)
+ one.put(to_put)
+ self.assertTrue(one._hard_link)
+ self.assertEqual(link.call_count, 2)
+ self.assertEqual(copy.call_count, 0)
+ # if link now fails, we fall back to copy
+ link.reset_mock()
+ link.side_effect = Exception('boom')
+ one.put(to_put)
+ self.assertFalse(one._hard_link)
+ self.assertEqual(link.call_count, 1)
+ self.assertEqual(copy.call_count, 2)
+
+
+class TestPutLocalNoHardlink(TestPutLocalMixin, TestCase):
+ """Run all TestPutLocalMixin tests with hard linking disabled"""
+
+ def make_one(self):
+ one = TestPutLocalMixin.make_one(self)
+ self.assertTrue(one._hard_link)
+ one._hard_link = False
+ return one
+
+
class TestPutS3(TestCase):
@patch("van.static.cdn._PutS3._get_imports")

0 comments on commit b755f30

Please sign in to comment.