Merge pull request #103 from psi29a/gridfs_coverage

dropping support python <= 2.5, small tidy-ups, expanding test coverage

tests/test_objects.py

@@ -12,18 +12,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import time
 from StringIO import StringIO
 
 from bson import objectid, timestamp
 import txmongo
 from txmongo import database
 from txmongo import collection
-from txmongo import gridfs
+from txmongo.gridfs import GridFS, GridIn, GridOut, GridOutIterator, errors
 from txmongo import filter as qf
-from txmongo._gridfs import GridIn
 from twisted.trial import unittest
 from twisted.internet import base, defer
+from twisted import _version
 
 mongo_host = "127.0.0.1"
 mongo_port = 27017
@@ -138,27 +139,126 @@ def _disconnect(self, conn):
         """ Disconnect the connection """
         yield conn.disconnect()
 
+    @defer.inlineCallbacks
+    def test_GridFileObjects(self):
+        """ Tests gridfs objects """
+        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
+        db = conn.test
+        db.fs.files.remove({})  # drop all objects there first
+        db.fs.chunks.remove({})
+        _ = GridFS(db)  # Default collection
+        with GridIn(db.fs, filename="test_with", contentType="text/plain", chunk_size=1024):
+            pass
+        grid_in_file = GridIn(db.fs, filename="test_1", contentType="text/plain",
+                              content_type="text/plain", chunk_size=2**2**2**2)
+        self.assertFalse(grid_in_file.closed)
+        if _version.version.major >= 15:
+            with self.assertRaises(TypeError):
+                yield grid_in_file.write(1)
+            with self.assertRaises(TypeError):
+                yield grid_in_file.write(u"0xDEADBEEF")
+            with self.assertRaises(AttributeError):
+                _ = grid_in_file.test
+        grid_in_file.test = 1
+        yield grid_in_file.write("0xDEADBEEF")
+        yield grid_in_file.write("0xDEADBEEF"*1048576)
+        fake_doc = {"_id": "test_id", "length": 1048576}
+        grid_out_file = GridOut(db.fs, fake_doc)
+        if _version.version.major >= 15:
+            with self.assertRaises(AttributeError):
+                _ = grid_out_file.testing
+        self.assertEqual(0, grid_out_file.tell())
+        grid_out_file.seek(1024)
+        self.assertEqual(1024, grid_out_file.tell())
+        grid_out_file.seek(1024, os.SEEK_CUR)
+        self.assertEqual(2048, grid_out_file.tell())
+        grid_out_file.seek(0, os.SEEK_END)
+        self.assertEqual(1048576, grid_out_file.tell())
+        self.assertRaises(IOError, grid_out_file.seek, 0, 4)
+        self.assertRaises(IOError, grid_out_file.seek, -1)
+        self.assertTrue("'_id': 'test_id'" in repr(grid_out_file))
+        yield grid_in_file.writelines(["0xDEADBEEF", "0xDEADBEAF"])
+        yield grid_in_file.close()
+        if _version.version.major >= 15:
+            with self.assertRaises(AttributeError):
+                grid_in_file.test = 2
+        self.assertEqual(1, grid_in_file.test)
+        if _version.version.major >= 15:
+            with self.assertRaises(AttributeError):
+                _ = grid_in_file.test_none
+        self.assertTrue(grid_in_file.closed)
+        if _version.version.major >= 15:
+            with self.assertRaises(ValueError):
+                yield grid_in_file.write("0xDEADBEEF")
+        yield conn.disconnect()
+
     @defer.inlineCallbacks
     def test_GridFsObjects(self):
         """ Tests gridfs objects """
         conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
         db = conn.test
-
-        gfs = gridfs.GridFS(db)  # Default collection
-
-        _ = GridIn(db.fs, filename="test", contentType="text/plain",
-                   chunk_size=2**2**2**2)
-        _ = gfs.new_file(filename="test2", contentType="text/plain",
-                         chunk_size=2**2**2**2)
-
+        db.fs.files.remove({})  # drop all objects there first
+        db.fs.chunks.remove({})
+        gfs = GridFS(db)  # Default collection
+        yield gfs.delete(u"test")
+
+        _ = gfs.new_file(filename="test_1", contentType="text/plain", chunk_size=65536)
+        yield conn.disconnect()
+        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
+        db = conn.test
+        db.fs.files.remove({})  # drop all objects there first
+        gfs = GridFS(db)  # Default collection
+        _ = yield gfs.put("0xDEADBEEF", filename="test_2", contentType="text/plain",
+                          chunk_size=65536)
         # disconnect
         yield conn.disconnect()
-
+
+        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
+        db = conn.test
+        gfs = GridFS(db)  # Default collection
+        _ = yield gfs.get("test_3")
+        # disconnect
+        yield conn.disconnect()
+
+    @defer.inlineCallbacks
+    def test_GridFsIteration(self):
+        """ Tests gridfs iterator """
+        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
+        db = conn.test
+        db.fs.files.remove({})  # drop all objects there first
+        db.fs.chunks.remove({})
+        gfs = GridFS(db)  # Default collection
+        new_file = gfs.new_file(filename="testName", contentType="text/plain", length=1048576,
+                                chunk_size=4096)
+        yield new_file.write("0xDEADBEEF"*4096*2)
+        yield new_file.close()
+        fake_doc = {"_id": new_file._id, "name": "testName", "length": 4096*2, "chunkSize": 4096,
+                    "contentType": "text/plain"}
+        grid_out_file = GridOut(db.fs, fake_doc)
+        iterator = GridOutIterator(grid_out_file, db.fs.chunks)
+        next_it = yield iterator.next()
+        self.assertEqual(len(next_it), 4096)
+        _ = yield iterator.next()
+        next_it = yield iterator.next()
+        self.assertEqual(next_it, None)
+
+        fake_bad_doc = {"_id": "bad_id", "name": "testName", "length": 4096*2,
+                        "chunkSize": 4096, "contentType": "text/plain"}
+        grid_bad_out_file = GridOut(db.fs, fake_bad_doc)
+        bad_iterator = GridOutIterator(grid_bad_out_file, db.fs.chunks)
+        if _version.version.major >= 15:
+            with self.assertRaises(errors.CorruptGridFile):
+                next_it = yield bad_iterator.next()
+
+        # disconnect
+        yield conn.disconnect()
+
     @defer.inlineCallbacks
     def test_GridFsOperations(self):
         """ Tests gridfs operations """
         conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
         db = conn.test
+        db.fs.files.remove({})  # Drop files first TODO: iterate through files and delete them
 
         # Don't forget to disconnect
         self.addCleanup(self._disconnect, conn)
@@ -172,9 +272,9 @@ def test_GridFsOperations(self):
 
         try:
             # Tests writing to a new gridfs file
-            gfs = gridfs.GridFS(db)  # Default collection
+            gfs = GridFS(db)  # Default collection
             g_in = gfs.new_file(filename="optest", contentType="text/plain",
-                                chunk_size=2**2**2**2)  # non-default chunk size used
+                                chunk_size=65536)  # non-default chunk size used
             # yielding to ensure writes complete before we close and close before we try to read
             yield g_in.write(in_file.read())
             yield g_in.close()

txmongo/_gridfs/__init__.py

@@ -18,15 +18,13 @@
 :mod:`pymongo`, exposing a file-like interface.
 """
 from twisted.internet import defer
-from txmongo._gridfs.errors import (NoFile,
-                                    UnsupportedAPI)
-from txmongo._gridfs.grid_file import (GridIn,
-                                       GridOut)
+from txmongo._gridfs.errors import NoFile
+from txmongo._gridfs.grid_file import GridIn, GridOut, GridOutIterator
 from txmongo import filter
-from txmongo.filter import (ASCENDING,
-                            DESCENDING)
+from txmongo.filter import ASCENDING, DESCENDING
 from txmongo.database import Database
 
+assert GridOutIterator
 
 class GridFS(object):
     """An instance of GridFS on top of a single Database.
@@ -180,21 +178,4 @@ def list(self):
         .. versionchanged:: 1.6
            Removed the `collection` argument.
         """
-        return self.__files.distinct("filename")
-
-    def open(self, *args, **kwargs):
-        """No longer supported.
-
-        .. versionchanged:: 1.6
-           The open method is no longer supported.
-        """
-        raise UnsupportedAPI("The open method is no longer supported.")
-
-    def remove(self, *args, **kwargs):
-        """No longer supported.
-
-        .. versionchanged:: 1.6
-           The remove method is no longer supported.
-        """
-        raise UnsupportedAPI("The remove method is no longer supported. "
-                             "Please use the delete method instead.")
+        return self.__files.distinct("filename")

txmongo/_gridfs/grid_file.py

@@ -16,29 +16,18 @@
 
 import datetime
 import math
-
 import os
 
-
 try:
     from cStringIO import StringIO
 except ImportError:
     from StringIO import StringIO
 
 from twisted.internet import defer
-from txmongo._gridfs.errors import CorruptGridFile, UnsupportedAPI
+from txmongo._gridfs.errors import CorruptGridFile
 from bson import Binary, ObjectId
 from txmongo.collection import Collection
 
-try:
-    _SEEK_SET = os.SEEK_SET
-    _SEEK_CUR = os.SEEK_CUR
-    _SEEK_END = os.SEEK_END
-except AttributeError:  # before 2.5
-    _SEEK_SET = 0
-    _SEEK_CUR = 1
-    _SEEK_END = 2
-
 """Default chunk size, in bytes."""
 DEFAULT_CHUNK_SIZE = 256 * 1024
 
@@ -61,7 +50,7 @@ def setter(self, value):
         self._file[field_name] = value
 
     if read_only:
-        docstring = docstring + "\n\nThis attribute is read-only."
+        docstring += "\n\nThis attribute is read-only."
     elif not closed_only:
         docstring = "%s\n\n%s" % (docstring, "This attribute can only be "
                                              "set before :meth:`close` has been called.")
@@ -241,7 +230,7 @@ def write(self, data):
                     data = data.encode(self.encoding)
                 except AttributeError:
                     raise TypeError("must specify an encoding for file in "
-                                    "order to write %s" % (data.__name__,))
+                                    "order to write %s" % data)
             read = StringIO(data).read
 
         if self._buffer.tell() > 0:
@@ -374,7 +363,7 @@ def tell(self):
         """
         return self.__position
 
-    def seek(self, pos, whence=_SEEK_SET):
+    def seek(self, pos, whence=os.SEEK_SET):
         """Set the current position of this file.
 
         :Parameters:
@@ -386,11 +375,11 @@ def seek(self, pos, whence=_SEEK_SET):
            to the current position, :attr:`os.SEEK_END` (``2``) to
            seek relative to the file's end.
         """
-        if whence == _SEEK_SET:
+        if whence == os.SEEK_SET:
             new_pos = pos
-        elif whence == _SEEK_CUR:
+        elif whence == os.SEEK_CUR:
             new_pos = self.__position + pos
-        elif whence == _SEEK_END:
+        elif whence == os.SEEK_END:
             new_pos = int(self.length) + pos
         else:
             raise IOError(22, "Invalid value for `whence`")
@@ -404,10 +393,6 @@ def close(self):
         self.__buffer = ''
         self.__current_chunk = -1
 
-    def __iter__(self):
-        """Deprecated."""
-        raise UnsupportedAPI("Iterating is deprecated for iterated reading")
-
     def __repr__(self):
         return str(self._file)
 
@@ -433,15 +418,3 @@ def next(self):
             raise CorruptGridFile("no chunk #%d" % self.__current_chunk)
         self.__current_chunk += 1
         defer.returnValue(str(chunk["data"]))
-
-
-class GridFile(object):
-    """No longer supported.
-
-    .. versionchanged:: 1.6
-       The GridFile class is no longer supported.
-    """
-
-    def __init__(self, *args, **kwargs):
-        raise UnsupportedAPI("The GridFile class is no longer supported. "
-                             "Please use GridIn or GridOut instead.")

txmongo/gridfs.py

@@ -2,8 +2,11 @@
 # Use of this source code is governed by the Apache License that can be
 # found in the LICENSE file.
 
-from txmongo._gridfs import GridFS, errors, grid_file
+from txmongo._gridfs import GridFS, GridIn, GridOut, errors, grid_file, GridOutIterator
 
 assert GridFS
+assert GridIn
+assert GridOut
+assert GridOutIterator
 assert errors
 assert grid_file