Merge branch 'safe-support'

tests/tests.py

@@ -50,7 +50,19 @@ def test_open_(self):
         with open(path) as fh:
             with open_(fh, compression=False) as fh2:
                 self.assertEqual(fh2.read(), 'foo')
-
+
+    def test_open_safe(self):
+        with self.assertRaises(IOError):
+            with open_('foobar', mode='r', errors=True) as fh:
+                pass
+        with self.assertRaises(ValueError):
+            with open_(None, mode='r', errors=True) as fh:
+                pass
+        with open_('foobar', mode='r', errors=False) as fh:
+            self.assertIsNone(fh)
+        with open_(None, mode='r', errors=False) as fh:
+            self.assertIsNone(fh)
+
     def test_xopen_invalid(self):
         # invalid path
         with self.assertRaises(ValueError):

tests/testutils.py

@@ -13,40 +13,45 @@ def setUp(self):
     def tearDown(self):
         self.root.close()
 
-    def test_safe_read(self):
-        self.assertEqual(safe_read('foobar'), '')
-        self.assertListEqual(list(safe_iter('foobar')), [])
+    def test_iter_lines(self):
+        self.assertListEqual(list(iter_lines('foobar', errors=False)), [])
 
         path = self.root.make_file()
         with open(path, 'wt') as o:
             o.write("1\n2\n3")
-        self.assertEqual(safe_read(path), "1\n2\n3")
         self.assertListEqual(
-            list(safe_iter(path)),
+            list(iter_lines(path)),
             ['1','2','3'])
         self.assertListEqual(
-            list(safe_iter(path, convert=int)),
+            list(iter_lines(path, convert=int)),
             [1,2,3])
 
     def test_read_chunked(self):
+        self.assertListEqual([], list(chunked_iter('foobar', errors=False)))
         path = self.root.make_file()
         with open(path, 'wt') as o:
             o.write("1234567890")
         chunks = list(chunked_iter(path, 3))
         self.assertListEqual([b'123', b'456', b'789', b'0'], chunks)
 
     def test_write_iterable(self):
+        linesep_len = len(os.linesep)
         path = self.root.make_file()
-        write_iterable(['foo'], path, linesep=None)
-        self.assertEqual(safe_read(path), 'foo')
+        self.assertEquals(3, write_iterable(['foo'], path, linesep=None))
+        self.assertEqual(list(iter_lines(path)), ['foo'])
         path = self.root.make_file()
-        write_iterable(('foo', 'bar', 'baz'), path, linesep=None)
+        self.assertEquals(
+            9 + (2*linesep_len),
+            write_iterable(('foo', 'bar', 'baz'), path, linesep=None))
         self.assertEqual(
-            safe_read(path),
-            os.linesep.join(('foo','bar','baz')))
+            list(iter_lines(path)),
+            ['foo','bar','baz'])
         path = self.root.make_file()
-        write_iterable(('foo', 'bar', 'baz'), path, linesep='|')
-        self.assertEqual(safe_read(path), 'foo|bar|baz')
+        self.assertEquals(
+            11, write_iterable(('foo', 'bar', 'baz'), path, linesep='|'))
+        self.assertEqual(list(iter_lines(path)), ['foo|bar|baz'])
+        path = self.root.make_file(mode='r')
+        self.assertEqual(-1, write_iterable(['foo'], path, errors=False))
 
     def test_read_dict(self):
         path = self.root.make_file()
@@ -64,10 +69,12 @@ def test_write_dict(self):
         path = self.root.make_file()
         write_dict(OrderedDict([('foo',1), ('bar',2)]), path, linesep=None)
         self.assertEqual(
-            safe_read(path),
-            os.linesep.join(('foo=1','bar=2')))
+            list(iter_lines(path)),
+            ['foo=1','bar=2'])
 
     def test_tsv(self):
+        self.assertListEqual([], list(delimited_file_iter('foobar', errors=False)))
+
         path = self.root.make_file()
         with open(path, 'wt') as o:
             o.write('a\tb\tc\n')
@@ -229,12 +236,17 @@ def test_uncompress_file_compression(self):
             self.assertEqual(i.read(), 'foo')
 
     def test_linecount(self):
+        self.assertEqual(-1, linecount('foobar', errors=False))
         path = self.root.make_file()
         with open(path, 'wt') as o:
             for i in range(100):
                 o.write(random_text())
                 if i != 99:
                     o.write('\n')
+        with self.assertRaises(ValueError):
+            linecount(path, buffer_size=-1)
+        with self.assertRaises(ValueError):
+            linecount(path, mode='wb')
         self.assertEqual(100, linecount(path))
 
     def test_linecount_empty(self):

xphyle/__init__.py

@@ -63,16 +63,20 @@ def guess_file_format(path : 'str') -> 'str':
     return fmt
 
 @contextmanager
-def open_(f, mode : 'str' = 'r', **kwargs):
+def open_(f, mode : 'str' = 'r', errors : 'bool' = True, **kwargs):
     """Context manager that frees you from checking if an argument is a path
     or a file object. Calls ``xopen`` to open files.
     
     Args:
         f: A path or file-like object
+        mode: The file open mode
+        errors: Whether to raise an error if there is a problem opening the
+            file. If False, yields None when there is an error.
         kwargs: Additional args to pass through to xopen (if ``f`` is a path)
     
     Yields:
-        A file-like object
+        A file-like object, or None if ``errors`` is False and there is a
+        problem opening the file.
     
     Examples:
         with open_('myfile') as infile:
@@ -84,8 +88,16 @@ def open_(f, mode : 'str' = 'r', **kwargs):
     """
     if isinstance(f, str):
         kwargs['context_wrapper'] = True
-        with xopen(f, mode, **kwargs) as fp:
-            yield fp
+        try:
+            with xopen(f, mode, **kwargs) as fp:
+                yield fp
+        except IOError:
+            if errors:
+                raise
+            else:
+                yield None
+    elif f is None and errors:
+        raise ValueError("'f' cannot be None")
     else:
         yield f
 

xphyle/formats.py

@@ -454,7 +454,7 @@ def open_file_python(self, f, mode : 'str', **kwargs):
 
     def compress_file(self, source, dest=None, keep : 'bool' = True,
                       compresslevel : 'int' = None,
-                      use_system : 'bool' = True) -> 'str':
+                      use_system : 'bool' = True, **kwargs) -> 'str':
         """Compress data from one file and write to another.
         
         Args:
@@ -464,6 +464,8 @@ def compress_file(self, source, dest=None, keep : 'bool' = True,
             keep: Whether to keep the source file
             compresslevel: Compression level
             use_system: Whether to try to use system-level compression
+            kwargs: Additional arguments to pass to the open method when opening
+                the destination file
         
         Returns:
             Path to the destination file
@@ -498,7 +500,7 @@ def compress_file(self, source, dest=None, keep : 'bool' = True,
                 p.communicate()
             else:
                 source_file = open(source, 'rb') if source_is_path else source
-                dest_file = self.open_file_python(dest, 'wb')
+                dest_file = self.open_file_python(dest, 'wb', **kwargs)
                 try:
                     # Perform sequential compression as the source
                     # file might be quite large
@@ -519,7 +521,7 @@ def compress_file(self, source, dest=None, keep : 'bool' = True,
         return dest
 
     def uncompress_file(self, source, dest=None, keep : 'bool' = True,
-                        use_system : 'bool' = True) -> 'str':
+                        use_system : 'bool' = True, **kwargs) -> 'str':
         """Uncompress data from one file and write to another.
         
         Args:
@@ -528,6 +530,8 @@ def uncompress_file(self, source, dest=None, keep : 'bool' = True,
                 If None, the file name is determined from ``source``.
             keep: Whether to keep the source file
             use_system: Whether to try to use system-level compression
+            kwargs: Additional arguments to passs to the open method when
+                opening the compressed file
         
         Returns:
             Path to the destination file
@@ -544,7 +548,9 @@ def uncompress_file(self, source, dest=None, keep : 'bool' = True,
         if dest is None:
             in_place = True
             if len(source_parts) > 2:
-                dest = os.path.join(*source_parts[0:2]) + ''.join(source_parts[2:-1])
+                dest = (
+                    os.path.join(*source_parts[0:2]) +
+                    ''.join(source_parts[2:-1]))
             else:
                 raise Exception("Cannot determine path for uncompressed file")
         dest_is_path = isinstance(dest, str)
@@ -560,7 +566,7 @@ def uncompress_file(self, source, dest=None, keep : 'bool' = True,
                 p = wrap_subprocess(cmd, stdin=psrc, stdout=dest_file)
                 p.communicate()
             else:
-                source_file = self.open_file_python(source, 'rb')
+                source_file = self.open_file_python(source, 'rb', **kwargs)
                 try:
                     # Perform sequential decompression as the source
                     # file might be quite large