Custom tar parser: end of record

leogao2 · Sep 18, 2020 · 83612b9 · 83612b9
1 parent 5b17330
commit 83612b9
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/lm_dataformat/__init__.py b/lm_dataformat/__init__.py
@@ -78,6 +78,9 @@ def read_tgz(self, file):
 
             # https://www.gnu.org/software/tar/manual/html_node/Standard.html
             # end at 135 not 136 because of \0 terminator
+            if hdr[124:135] == b'\0'*11:
+                # end of record
+                break
             size = int(hdr[124:135], 8)
 
             padded_size = ceil(size / 512) * 512

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="lm_dataformat", # Replace with your own username
-    version="0.0.12",
+    version="0.0.13",
     author="Leo Gao",
     author_email="leogao31@gmail.com",
     description="A utility for storing and reading files for LM training.",