Optionally use crc32c library (#429)

* Optionally use crc32c library

- If crc32c library with native implementation is not presented, use the current implementation.
- If crc32c library is presented try to use SSE instructions or a software implementation in C ("auto" mode)
- Don't add crc32c to the requirements list to make the dependency optional.

* Fix bug with using native crc32c implementation

- Use native crc32c without masking because it will be done by caller
- Add tests
- Add crc32c as a testing dependency

* Rewrite tests for crc32c module

.travis.yml

@@ -51,6 +51,7 @@ install:
   - pip install boto3
   - pip install moto
   - pip install visdom
+  - pip install crc32c
   - conda install ffmpeg
   - conda list
   - python -c "import imageio; imageio.plugins.ffmpeg.download()"

README.md

@@ -25,6 +25,8 @@ or build from source:
 
 `git clone https://github.com/lanpa/tensorboardX && cd tensorboardX && python setup.py install`
 
+You can optionally install [`crc32c`](https://github.com/ICRAR/crc32c) to speed up saving a large amount of data.
+
 
 ## Example
 

setup.py

@@ -51,7 +51,8 @@ def run(self):
 
 test_requirements = [
     'pytest',
-    'matplotlib'
+    'matplotlib',
+    'crc32c',
 ]
 
 setup(

tensorboardX/crc32c.py

@@ -1,5 +1,13 @@
 # https://www.ietf.org/rfc/rfc3309.txt
 import array
+import os
+
+try:
+    if os.environ.get('CRC32C_SW_MODE', None) is None:
+        os.environ['CRC32C_SW_MODE'] = 'auto'
+    from crc32c import crc32 as _crc32c_native
+except ImportError:
+    _crc32c_native = None
 
 
 CRC_TABLE = (
@@ -69,7 +77,6 @@
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,
 )
 
-
 CRC_INIT = 0
 
 _MASK = 0xFFFFFFFF
@@ -112,7 +119,7 @@ def crc_finalize(crc):
     return crc & _MASK
 
 
-def crc32c(data):
+def _crc32c(data):
     """Compute CRC-32C checksum of the data.
 
     Args:
@@ -122,3 +129,6 @@ def crc32c(data):
       32-bit CRC-32C checksum of data as long.
     """
     return crc_finalize(crc_update(CRC_INIT, data))
+
+
+crc32c = _crc32c if _crc32c_native is None else _crc32c_native

tests/test_crc32c.py

@@ -0,0 +1,18 @@
+import unittest
+from tensorboardX.crc32c import _crc32c, _crc32c_native, crc32c
+
+
+class CRC32CTest(unittest.TestCase):
+    def test_crc32c(self):
+        data = b'abcd'
+        assert crc32c(data) == 0x92c80a31
+
+    def test_crc32c_python(self):
+        data = b'abcd'
+        assert _crc32c(data) == 0x92c80a31
+
+    def test_crc32c_native(self):
+        if _crc32c_native is None:
+            return
+        data = b'abcd'
+        assert _crc32c_native(data) == 0x92c80a31