Start up compat and encoding files

There were a few things which were being reused consistently for
compatibility purposes, such as the `buffer`/`memoryview` functions
as well as the `izip` method which needed to be aliased for
Python 3.  The `buffer` function was taken from `smmap` [1] and
reworked slightly to handle the optional third parameter.

This also adds a compatibility file dedicated entirely to encoding
issues, which seem to be the biggest problem.  The main functions
were taken in part from the Django project [2] and rewritten
slightly because our needs are a bit more narrow.

A constants file has been added to consistently handle the
constants which are required for the gitdb project in the core
and the tests.  This is part of a greater plan to reorganize
the `util.py` file included in this project.

This points the async extension back at the original repository
and points it to the latest commit.

[1]: https://github.com/Byron/smmap/blob/1af4b42a2354acbb53c7956d647655922658fd80/smmap/util.py#L20-L26
[2]: https://github.com/django/django/blob/b8d255071ead897cf68120cd2fae7c91326ca2cc/django/utils/encoding.py

Author: kevin-brown

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "async"]
 	path = gitdb/ext/async
-	url = https://github.com/kevin-brown/async.git
+	url = https://github.com/gitpython-developers/async.git
 [submodule "smmap"]
 	path = gitdb/ext/smmap
 	url = https://github.com/Byron/smmap.git

gitdb/const.py

@@ -0,0 +1,5 @@
+from gitdb.utils.encoding import force_bytes
+
+NULL_BYTE = force_bytes("\0")
+NULL_HEX_SHA = "0" * 40
+NULL_BIN_SHA = NULL_BYTE * 20

gitdb/db/base.py

@@ -178,15 +178,9 @@ def db_path(self, rela_path):
         """
         :return: the given relative path relative to our database root, allowing
             to pontentially access datafiles"""
-        if sys.version_info[0] == 3:
-            text_type = str
-        else:
-            text_type = basestring
-
-        if not isinstance(rela_path, text_type):
-            rela_path = rela_path.decode("utf-8")
+        from gitdb.utils.encoding import force_text
 
-        return join(self._root_path, rela_path)
+        return join(self._root_path, force_text(rela_path))
     #} END interface
 
 

gitdb/db/loose.py

@@ -51,6 +51,8 @@
     stream_copy
 )
 
+from gitdb.utils.compat import MAXSIZE
+
 import tempfile
 import mmap
 import sys
@@ -200,7 +202,7 @@ def store(self, istream):
                 if istream.binsha is not None:
                     # copy as much as possible, the actual uncompressed item size might
                     # be smaller than the compressed version
-                    stream_copy(istream.read, writer.write, sys.maxint, self.stream_chunk_size)
+                    stream_copy(istream.read, writer.write, MAXSIZE, self.stream_chunk_size)
                 else:
                     # write object with header, we have to make a new one
                     write_object(istream.type, istream.size, istream.read, writer.write,

gitdb/ext/async

@@ -16,10 +16,7 @@
 import mmap
 from itertools import islice
 
-try:
-    from itertools import izip
-except ImportError:
-    izip = zip
+from gitdb.utils.compat import izip
 
 from io import StringIO
 
@@ -394,10 +391,14 @@ def loose_object_header_info(m):
     :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the
         object as well as its uncompressed size in bytes.
     :param m: memory map from which to read the compressed object data"""
+    from gitdb.const import NULL_BYTE
+    from gitdb.utils.encoding import force_text
+
     decompress_size = 8192      # is used in cgit as well
     hdr = decompressobj().decompress(m, decompress_size)
-    type_name, size = hdr[:hdr.find("\0".encode("ascii"))].split(" ".encode("ascii"))
-    return type_name.decode("ascii"), int(size)
+    type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii"))
+
+    return force_text(type_name), int(size)
 
 def pack_object_header_info(data):
     """

gitdb/fun.py

@@ -63,10 +63,8 @@
 
 from binascii import crc32
 
-try:
-    from itertools import izip
-except ImportError:
-    izip = zip
+from gitdb.const import NULL_BYTE
+from gitdb.utils.compat import izip, buffer
 
 import tempfile
 import array
@@ -90,6 +88,8 @@ def pack_object_at(cursor, offset, as_stream):
     :parma offset: offset in to the data at which the object information is located
     :param as_stream: if True, a stream object will be returned that can read
         the data, otherwise you receive an info object only"""
+    from gitdb.utils.encoding import force_bytes
+
     data = cursor.use_region(offset).buffer()
     type_id, uncomp_size, data_rela_offset = pack_object_header_info(data)
     total_rela_offset = None                # set later, actual offset until data stream begins
@@ -120,16 +120,12 @@ def pack_object_at(cursor, offset, as_stream):
     # END handle type id
     abs_data_offset = offset + total_rela_offset
     if as_stream:
-        try:
-            buff = memoryview(data)[total_rela_offset:].tobytes()
-        except (NameError, TypeError):
-            buff = buffer(data, total_rela_offset)
+        buff = buffer(data, total_rela_offset)
         stream = DecompressMemMapReader(buff, False, uncomp_size)
         if delta_info is None:
             return abs_data_offset, OPackStream(offset, type_id, uncomp_size, stream)
         else:
-            if hasattr(delta_info, "tobytes"):
-                delta_info = delta_info.tobytes()
+            delta_info = force_bytes(delta_info)
             return abs_data_offset, ODeltaPackStream(offset, type_id, uncomp_size, delta_info, stream)
     else:
         if delta_info is None:
@@ -468,7 +464,7 @@ def partial_sha_to_index(self, partial_bin_sha, canonical_length):
         hi = self._fanout_table[first_byte]     # the upper, right bound of the bisection
 
         # fill the partial to full 20 bytes
-        filled_sha = partial_bin_sha + '\0'.encode("ascii") * (20 - len(partial_bin_sha))
+        filled_sha = partial_bin_sha + NULL_BYTE * (20 - len(partial_bin_sha))
 
         # find lowest
         while lo < hi:

gitdb/pack.py

@@ -27,6 +27,10 @@
     zlib
 )
 
+from gitdb.const import NULL_BYTE
+from gitdb.utils.compat import buffer
+from gitdb.utils.encoding import force_bytes, force_text
+
 has_perf_mod = False
 try:
     from _perf import apply_delta as c_apply_delta
@@ -99,7 +103,7 @@ def _parse_header_info(self):
         maxb = 512              # should really be enough, cgit uses 8192 I believe
         self._s = maxb
         hdr = self.read(maxb)
-        hdrend = hdr.find("\0".encode("ascii"))
+        hdrend = hdr.find(NULL_BYTE)
         typ, size = hdr[:hdrend].split(" ".encode("ascii"))
         size = int(size)
         self._s = size
@@ -113,7 +117,7 @@ def _parse_header_info(self):
 
         self._phi = True
 
-        return typ.decode("ascii"), size
+        return force_text(typ), size
 
     #{ Interface
 
@@ -264,10 +268,7 @@ def read(self, size=-1):
         # END adjust winsize
 
         # takes a slice, but doesn't copy the data, it says ...
-        try:
-            indata = memoryview(self._m)[self._cws:self._cwe].tobytes()
-        except (NameError, TypeError):
-            indata = buffer(self._m, self._cws, self._cwe - self._cws)
+        indata = buffer(self._m, self._cws, self._cwe - self._cws)
 
         # get the actual window end to be sure we don't use it for computations
         self._cwe = self._cws + len(indata)
@@ -379,6 +380,7 @@ def _set_cache_too_slow_without_c(self, attr):
 
     def _set_cache_brute_(self, attr):
         """If we are here, we apply the actual deltas"""
+        from gitdb.utils.compat import buffer
 
         # TODO: There should be a special case if there is only one stream
         # Then the default-git algorithm should perform a tad faster, as the
@@ -546,7 +548,10 @@ def __init__(self):
     def write(self, data):
         """:raise IOError: If not all bytes could be written
         :return: length of incoming data"""
+
+        data = force_bytes(data)
         self.sha1.update(data)
+
         return len(data)
 
     # END stream interface
@@ -589,8 +594,11 @@ def __getattr__(self, attr):
         return getattr(self.buf, attr)
 
     def write(self, data):
+        data = force_bytes(data)
+
         alen = Sha1Writer.write(self, data)
         self.buf.write(self.zip.compress(data))
+
         return alen
 
     def close(self):
@@ -630,11 +638,15 @@ def __init__(self, fd):
     def write(self, data):
         """:raise IOError: If not all bytes could be written
         :return: lenght of incoming data"""
+        data = force_bytes(data)
+
         self.sha1.update(data)
         cdata = self.zip.compress(data)
         bytes_written = write(self.fd, cdata)
+
         if bytes_written != len(cdata):
             raise self.exc
+
         return len(data)
 
     def close(self):

gitdb/stream.py

@@ -35,7 +35,7 @@ class TestDBBase(TestBase):
     """Base class providing testing routines on databases"""
 
     # data
-    two_lines = "1234\nhello world".encode("ascii")
+    two_lines = "1234\nhello world"
     all_data = (two_lines, )
 
     def _assert_object_writing_simple(self, db):
@@ -81,8 +81,7 @@ def _assert_object_writing(self, db):
 
                 prev_ostream = db.set_ostream(ostream)
                 assert type(prev_ostream) in ostreams or prev_ostream in ostreams
-
-                istream = IStream(str_blob_type, len(data), BytesIO(data))
+                istream = IStream(str_blob_type, len(data), BytesIO(data.encode("ascii")))
 
                 # store returns same istream instance, with new sha set
                 my_istream = db.store(istream)
@@ -131,8 +130,7 @@ def _assert_object_writing_async(self, db):
         """Test generic object writing using asynchronous access"""
         ni = 5000
         def istream_generator(offset=0, ni=ni):
-            for data_src in xrange(ni):
-                print(type(data_src), type(offset))
+            for data_src in range(ni):
                 data = bytes(data_src + offset)
                 yield IStream(str_blob_type, len(data), BytesIO(data))
             # END for each item

gitdb/test/db/lib.py

@@ -30,10 +30,7 @@
     mman = SlidingWindowMapManager()
 #END handle mman
 
-try:
-    import hashlib
-except ImportError:
-    import sha
+import hashlib
 
 try:
     from struct import unpack_from
@@ -84,9 +81,8 @@ def unpack_from(fmt, data, offset=0):
 close = os.close
 fsync = os.fsync
 
-# constants
-NULL_HEX_SHA = "0"*40
-NULL_BIN_SHA = "\0"*20
+# Backwards compatibility imports
+from gitdb.const import NULL_BIN_SHA, NULL_HEX_SHA
 
 #} END Aliases