Catch requests.exceptions.ChunkedEncodingError and similar

waybackpack/__init__.py

@@ -2,4 +2,4 @@
 from .asset import Asset
 from .pack import Pack
 from .cdx import search
-__version__ = "0.3.2"
+from .version import __version__

waybackpack/asset.py

@@ -44,6 +44,7 @@ def fetch(self,
         url = self.get_archive_url(raw)
         res = session.get(url)
         content = res.content
+        if content == None: return
 
         if raw:
             return content

waybackpack/cli.py

@@ -2,12 +2,18 @@
 from .session import Session
 from .pack import Pack
 from .cdx import search
+from .version import __version__
 from .settings import DEFAULT_USER_AGENT, DEFAULT_ROOT
 import argparse
 import logging
 
 def parse_args():
     parser = argparse.ArgumentParser()
+
+    parser.add_argument("--version",
+        action="version",
+        version="%(prog)s " + __version__)
+
     parser.add_argument("url",
         help="The URL of the resource you want to download.")
 
@@ -48,6 +54,10 @@ def parse_args():
     parser.add_argument("--collapse",
         help="An archive.org `collapse` parameter. Cf.: https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md#collapsing")
 
+    parser.add_argument("--ignore-errors",
+        help="Don't crash on non-HTTP errors e.g., the requests library's ChunkedEncodingError. Instead, log error and continue. Cf. https://github.com/jsvine/waybackpack/issues/19",
+        action="store_true")
+
     parser.add_argument("--quiet",
         action="store_true",
         help="Don't log progress to stderr.")
@@ -58,7 +68,10 @@ def parse_args():
 def main():
     args = parse_args()
 
-    logging.basicConfig(level=(logging.WARN if args.quiet else logging.INFO))
+    logging.basicConfig(
+        level=(logging.WARN if args.quiet else logging.INFO),
+        format="%(levelname)s:%(name)s: %(message)s"
+    )
 
     session = Session(
         user_agent=args.user_agent,
@@ -86,6 +99,7 @@ def main():
             args.dir,
             raw=args.raw,
             root=args.root,
+            ignore_errors=args.ignore_errors
         )
     else:
         flag = "id_" if args.raw else ""

waybackpack/pack.py

@@ -35,7 +35,8 @@ def __init__(self,
 
     def download_to(self, directory,
         raw=False,
-        root=DEFAULT_ROOT):
+        root=DEFAULT_ROOT,
+        ignore_errors=False):
 
         for asset in self.assets:
             path_head, path_tail = os.path.split(self.parsed_url.path)
@@ -57,11 +58,24 @@ def download_to(self, directory,
                     asset.timestamp)
             )
 
-            content = asset.fetch(
-                session=self.session,
-                raw=raw,
-                root=root
-            )
+            try:
+                content = asset.fetch(
+                    session=self.session,
+                    raw=raw,
+                    root=root
+                )
+            except Exception as e:
+                if ignore_errors == True:
+                    ex_name = ".".join([ e.__module__, e.__class__.__name__ ])
+                    logger.warn("ERROR -- {0} @ {1} -- {2}: {3}".format(
+                        asset.original_url,
+                        asset.timestamp,
+                        ex_name,
+                        e
+                    ))
+                    return
+                else:
+                    raise
 
             try:
                 os.makedirs(filedir)

waybackpack/session.py

@@ -10,13 +10,16 @@ def __init__(self, follow_redirects=False, user_agent=DEFAULT_USER_AGENT):
         self.user_agent = user_agent
 
     def get(self, url, **kwargs):
-        headers = { "User-Agent": self.user_agent }
+        headers = {
+            "User-Agent": self.user_agent,
+        }
         response_is_final = False
         while (response_is_final == False):
             res = requests.get(
                 url,
                 allow_redirects=self.follow_redirects,
                 headers=headers,
+                stream=True,
                 **kwargs
             )
             if res.status_code != 200:

waybackpack/version.py

@@ -0,0 +1 @@
+__version__ = "0.3.2"