Raise err if CDX API doesn't respect showDupeCount

h/t @CaptSolo in #81 ... and internetarchive/wayback#271
jsvine · May 15, 2024 · 9e896ad · 9e896ad
1 parent dca5076
commit 9e896ad
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/waybackpack/cdx.py b/waybackpack/cdx.py
@@ -43,6 +43,10 @@ def search(
     fields = cdx[0]
     snapshots = [dict(zip(fields, row)) for row in cdx[1:]]
     if uniques_only:
+        if len(snapshots) and "dupecount" not in snapshots[0]:
+            raise WaybackpackException(
+                "Wayback Machine CDX API not respecting showDupeCount=true; retry without --uniques-only."
+            )
         return [s for s in snapshots if int(s["dupecount"]) == 0]
     else:
         return snapshots