Add main, remove logging

.gitignore

@@ -6,3 +6,4 @@ __pycache__/
 /.direnv/
 
 /.cache/
+*.log

ascl_net_scraper/__main__.py

@@ -0,0 +1,24 @@
+import logging
+
+from tqdm import tqdm
+
+from .main import scrape_index_list, scrape_details
+
+logger = logging.getLogger("charmonium.freeze")
+logger.setLevel(logging.DEBUG)
+fh = logging.FileHandler("freeze.log")
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
+logger.addHandler(fh)
+
+logger = logging.getLogger("charmonium.cache.ops")
+logger.setLevel(logging.DEBUG)
+fh = logging.FileHandler("cache.log")
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
+logger.addHandler(fh)
+
+
+records = scrape_index_list(10)
+for record in tqdm(records, total=len(records)):
+    detailed_record = scrape_details(record.details_url)

ascl_net_scraper/main.py

@@ -20,23 +20,6 @@
 DEFAULT_PARSER = "html5lib"
 group = MemoizedGroup(fine_grain_persistence=True)
 
-# import logging
-
-# logger = logging.getLogger("charmonium.freeze")
-# logger.setLevel(logging.DEBUG)
-# fh = logging.FileHandler("freeze.log")
-# fh.setLevel(logging.DEBUG)
-# fh.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
-# logger.addHandler(fh)
-
-# logger = logging.getLogger("charmonium.cache.ops")
-# logger.setLevel(logging.DEBUG)
-# fh = logging.FileHandler("cache.log")
-# fh.setLevel(logging.DEBUG)
-# fh.setFormatter(logging.Formatter("%(asctime)s: %(message)s"))
-# logger.addHandler(fh)
-
-
 @dataclass
 class CodeRecord:
     ascl_id: Optional[Tuple[int, int]]