Fix network error when an incorrect download URL is outputted by the …

…Search API (#55) * Fix network error when an incorrect download URL is outputted by the Search API. * Remove coverage from exception handler.
jadchaar · Jan 23, 2021 · e5c26e5 · e5c26e5
1 parent f950221
commit e5c26e5
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,12 @@
 # Changelog
 
-## 4.0.0 - 1/16/2021
+## 4.0.1 - 1/23/2021
+
+### Fixed
+
+- Downloads will no longer halt prematurely if a filing document (full or detail) cannot be found (e.g. when the EDGAR Search API outputs incorrect download URLs). Now, the package will automatically catch such network errors, print a helpful warning message, and then proceed to download the remaining filings.
+
+## 4.0.0 - 1/17/2021
 
 This is a major breaking release. Please see the [v4 migration guide](https://github.com/jadchaar/sec-edgar-downloader/blob/master/docs/v4_migration_guide.md) for information on how to upgrade and adapt your existing codebase to the new package version.
 

diff --git a/sec_edgar_downloader/_utils.py b/sec_edgar_downloader/_utils.py
@@ -243,25 +243,37 @@ def download_filings(
     include_filing_details: bool,
 ) -> None:
     for filing in filings_to_fetch:
-        download_and_save_filing(
-            download_folder,
-            ticker_or_cik,
-            filing.accession_number,
-            filing_type,
-            filing.full_submission_url,
-            FILING_FULL_SUBMISSION_FILENAME,
-        )
-
-        if include_filing_details:
+        try:
             download_and_save_filing(
                 download_folder,
                 ticker_or_cik,
                 filing.accession_number,
                 filing_type,
-                filing.filing_details_url,
-                filing.filing_details_filename,
-                resolve_urls=True,
+                filing.full_submission_url,
+                FILING_FULL_SUBMISSION_FILENAME,
             )
+        except requests.exceptions.HTTPError as e:  # pragma: no cover
+            print(
+                "Skipping full submission download for "
+                f"'{filing.accession_number}' due to network error: {e}."
+            )
+
+        if include_filing_details:
+            try:
+                download_and_save_filing(
+                    download_folder,
+                    ticker_or_cik,
+                    filing.accession_number,
+                    filing_type,
+                    filing.filing_details_url,
+                    filing.filing_details_filename,
+                    resolve_urls=True,
+                )
+            except requests.exceptions.HTTPError as e:  # pragma: no cover
+                print(
+                    f"Skipping filing detail download for "
+                    f"'{filing.accession_number}' due to network error: {e}."
+                )
 
 
 def get_number_of_unique_filings(filings: List[FilingMetadata]) -> int:

diff --git a/sec_edgar_downloader/_version.py b/sec_edgar_downloader/_version.py
@@ -1 +1 @@
-__version__ = "4.0.0"
+__version__ = "4.0.1"
diff --git a/tests/test_network_errors.py b/tests/test_network_errors.py
@@ -0,0 +1,22 @@
+# Regression test for issue 54
+def test_http_error_on_download(downloader, capsys):
+    dl, _ = downloader
+
+    filing_type = "8-K"
+    cik = "0000101778"
+    # Search API outputs an incorrect filename for this filing's detail document,
+    # so we must catch the 404 HTTP error and continue execution normally.
+    dl.get(
+        filing_type,
+        cik,
+        download_details=True,
+        include_amends=True,
+        after="2001-02-26",
+        before="2001-02-28",
+    )
+
+    captured = capsys.readouterr()
+    # Full submission should download fine
+    assert "skipping full submission download" not in captured.out.lower()
+    # Filing detail download should fail
+    assert "skipping filing detail download" in captured.out.lower()