autopurge: add some logging and lower the parallel purges

facenapalmscripts/autopurge.py

@@ -13,31 +13,59 @@
 from pathlib import Path
 import pywikibot
 import pywikibot.exceptions
+import logging
 
 # manually load the auth
 curdir = Path(__file__).parent.parent.absolute()
 userfile = curdir / "user-config.py"
-exec(compile(userfile.read_text(), str(userfile), 'exec'), vars(pywikibot.config))
+exec(compile(userfile.read_text(), str(userfile), "exec"), vars(pywikibot.config))
 
-def process_purge(site, catname, limit=500):
+
+def process_purge(site, catname, limit=50):
     """Purge all pages from category and return status for logging."""
     members = list(pywikibot.Category(site, catname).members())
-    length = len(members);
+    logging.info(
+        "Starting purge: Processing %d members for category %s", len(members), catname
+    )
+    length = len(members)
     for i in range(0, length, limit):
-        if not site.purgepages(members[i:i+limit]):
-            return "неожиданный ответ сервера"
+        try:
+            if not site.purgepages(members[i : i + limit]):
+                return "неожиданный ответ сервера"
+        except Exception as error:
+            logging.error(
+                "Failed trying to purge members %d to %d for category %s: %s",
+                i,
+                i + limit,
+                catname,
+                str(error),
+            )
+            raise
+
+        logging.info("    purged from %d to %d", i, i + limit)
+    logging.info("Purged %d members", len(members))
     return str(length)
 
+
 def process_hourly(site):
     """Purge all hourly-purged pages and return log part."""
-    return "срочных: " + process_purge(site, "К:Википедия:Страницы с ежечасно очищаемым кэшем")
+    logging.info("Starting hourly purge...")
+    return "срочных: " + process_purge(
+        site, "К:Википедия:Страницы с ежечасно очищаемым кэшем"
+    )
+
 
 def process_daily(site):
     """Purge all daily-purged pages and return log part."""
-    return "ежедневных: " + process_purge(site, "К:Википедия:Страницы с ежедневно очищаемым кэшем")
+    logging.info("Starting daily purge...")
+    return "ежедневных: " + process_purge(
+        site, "К:Википедия:Страницы с ежедневно очищаемым кэшем"
+    )
+
 
 def process_null(site):
     """Purge all daily-nulledited pages and return log part."""
+    logging.info("Starting null purge...")
     catname = "К:Википедия:Страницы с ежедневно совершаемой нулевой правкой"
     members = list(pywikibot.Category(site, catname).members())
     errors = 0
@@ -51,22 +79,30 @@ def process_null(site):
             errors += 1
     return "нулевых правок: " + str(len(members) - errors)
 
+
 def log(site, respond):
     """Edit template status page."""
     page = pywikibot.Page(site, "Шаблон:Очищать кэш/статус")
-    page.text = "~~~~. Обработано " + "; ".join(respond) + "<noinclude>\n[[Категория:Шаблоны:Подстраницы шаблонов]]\n</noinclude>"
+    page.text = (
+        "~~~~. Обработано "
+        + "; ".join(respond)
+        + "<noinclude>\n[[Категория:Шаблоны:Подстраницы шаблонов]]\n</noinclude>"
+    )
     page.save("Отчёт.")
 
-KEYS = {
-    "--hourly": process_hourly,
-    "--daily": process_daily,
-    "--null": process_null
-}
+
+KEYS = {"--hourly": process_hourly, "--daily": process_daily, "--null": process_null}
 
 DISABLE_LOG = "--nolog"
 
+
 def main():
     """Get console arguments and call corresponding fucntions."""
+    logging.basicConfig(
+        # add timestamps to logs
+        format="%(asctime)s %(levelname)-8s %(message)s",
+        level=logging.INFO,
+    )
     if len(sys.argv) == 1:
         return
     args = sys.argv[1:]
@@ -79,5 +115,8 @@ def main():
     if DISABLE_LOG not in args:
         log(site, respond)
 
+    logging.info("Finished")
+
+
 if __name__ == "__main__":
     main()