propkb now doesn't download docs it should skip later

staffanm · Mar 13, 2018 · ed21bcc · ed21bcc
1 parent 36bd328
commit ed21bcc
Showing 1 changed file with 20 additions and 0 deletions.
diff --git a/ferenda/sources/legal/se/propositioner.py b/ferenda/sources/legal/se/propositioner.py
@@ -647,6 +647,24 @@ def get_default_options(cls):
         opts['ocr'] = False
         return opts
 
+    def download_get_first_page(self):
+        # if we have already successfully downloaded everything, there
+        # is no need to even make a single network request (and we'd
+        # have to do at least 100 otherwise) since no new docs will
+        # ever be published (normally -- and if they are, just set
+        # config.refresh)
+        if (not self.config.refresh and
+            'lastdownload' in self.config and
+            self.config.lastdownload):
+            class DummyResp(object):
+                def raise_for_status(self):
+                    pass
+                text = "<h1>no data</h1>"
+            return DummyResp()
+        else:
+            return super(PropKB, self).download_get_first_page()
+
+
     @decorators.downloadmax
     def download_get_basefiles(self, source):
         yielded = set()
@@ -669,6 +687,8 @@ def download_get_basefiles(self, source):
                         part = m.group("part")
                         if (basefile,part) in yielded:
                             continue
+                        if self.get_parse_options(basefile) == "skip":
+                            continue
                         if part and int(part) > 1:
                             # do something smart here so that
                             # download() will treat the yielded value