Merge pull request #26 from c2corg/split-check-process

Split check process into clean / report

campbot/__main__.py

@@ -2,8 +2,10 @@
 CampBot, Python bot framework for camptocamp.org
 
 Usage:
-  campbot check_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
+  campbot clean_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
+  campbot report_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
   campbot clean <url_or_file> <langs> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch] [--bbcode]
+  campbot report <url_or_file> <lang> [--login=<login>] [--password=<password>] [--delay=<seconds>]
   campbot contribs [--out=<filename>] [--starts=<start_date>] [--ends=<end_date>] [--delay=<seconds>]
   campbot export <url> [--out=<filename>] [--delay=<seconds>]
 
@@ -19,13 +21,15 @@
 
 
 Commands:
-  check_rc      Check (and clean) recent changes.
+  report_rc     Make quality report on recent changes.
+  clean_rc      Clean recent changes.
   clean         Clean documents.
                 <url_or_file> is like https://www.camptocamp.org/routes#a=523281, or, simplier, routes#a=523281. 
                 filename is also accepted, and must be like : 
                 123 | r
                 456 | w
                 <langs> is comma-separated lang identifiers, like fr,de for french and german.
+  report        Make quality report on documents.
   contribs      Export all contribution in a CSV file. <start_date> and <end_date> are like 2018-05-12
   export        Export all documents in a CSV file.
                 <url> is like https://www.camptocamp.org/outings#u=2, or, simplier, outings#u=2
@@ -67,15 +71,25 @@ def main_entry_point():
 
 
 def main(args):
-    if args["check_rc"]:
-        from campbot.checkers import check_recent_changes
+    if args["report_rc"]:
+        from campbot.checkers import report_recent_changes
 
-        check_recent_changes(
+        report_recent_changes(
             get_campbot(args),
             days=int(args["<days>"]),
             ask_before_saving=not args["--batch"],
         )
 
+    elif args["clean_rc"]:
+        get_campbot(args).clean_recent_changes(
+            days=int(args["<days>"]), lang="fr", ask_before_saving=not args["--batch"],
+        )
+
+    elif args["report"]:
+        get_campbot(args).report(
+            args["<url_or_file>"], lang=args["<lang>"],
+        )
+
     elif args["clean"]:
         get_campbot(args).clean(
             args["<url_or_file>"],

campbot/checkers.py

@@ -4,7 +4,7 @@
 """
 ## Test contributions validity, and report suspicious contribution.
 
-The main function is `check_recent_changes(bot, days)`, and it will test contributions made in the <days> past days. 
+The main function is `report_recent_changes(bot, days)`, and it will test contributions made in the <days> past days. 
 
 A test can inherits one of those two classes:
 
@@ -193,7 +193,7 @@ def get_report(self, bot, lang):
         return "\n".join(result)
 
 
-def check_recent_changes(bot, days, ask_before_saving):
+def report_recent_changes(bot, days, ask_before_saving):
     check_message_url = (
         "https://forum.camptocamp.org/t/topoguide-verifications-automatiques/201480"
     )
@@ -202,8 +202,6 @@ def check_recent_changes(bot, days, ask_before_saving):
     newest_date = utils.today().replace(hour=0, minute=0, second=0, microsecond=0)
     oldest_date = newest_date - datetime.timedelta(days=days)
 
-    bot.fix_recent_changes(oldest_date, newest_date, lang, ask_before_saving)
-
     tests = get_fixed_tests(lang)
     tests += get_re_tests(bot.forum.get_post(url=check_message_url), lang)
 
@@ -281,6 +279,16 @@ def get_fixed_tests(lang):
     ]
 
 
+def get_document_tests(lang):
+    """ Returns a list of tests that can be run on a single document"""
+
+    return [
+        HistoryTest(lang),
+        MainWaypointTest(),
+        RouteTypeTest(),
+    ]
+
+
 class _BaseTest(object):
     def __init__(self, name, lang=None):
         self.name = name

campbot/core.py

@@ -14,6 +14,7 @@
 from . import utils
 from . import objects
 from campbot.processors import get_automatic_replacments
+from campbot.checkers import get_document_tests
 
 try:
     # py2
@@ -534,6 +535,43 @@ def clean(self, url_or_filename, langs, ask_before_saving=True, clean_bbcode=Fal
             documents, processors, langs, ask_before_saving, excluded_ids=[996571,]
         )
 
+    def report(self, url_or_filename, lang):
+        """
+            Make quality report on a set of document.
+
+            :param url_or_filename: Camptocamp.org URL, or filename
+            :param langs: comma-separated list of lang identifiers
+        """
+
+        documents = [d for d in self.get_documents(url_or_filename)]
+
+        tests = get_document_tests(lang)
+        forum_report = []
+        stdout_report = []
+
+        for test in tests:
+            failing_docs = []
+            for document in documents:
+                if "redirects_to" in document:
+                    pass  # document id is not available...
+                elif not test.test_document(document):
+                    failing_docs.append(document)
+
+            if len(failing_docs) != 0:
+                stdout_report.append(test.name)
+                forum_report.append("* {}".format(test.name))
+
+                failing_docs.sort(key=lambda d: d.get_title(lang))
+
+                for document in failing_docs:
+                    url = document.get_url()
+                    title = document.get_title(lang)
+
+                    stdout_report.append("    {}\t {}".format(url, title))
+                    forum_report.append("  * [{}]({})".format(title, url))
+
+        print("\n".join(stdout_report))
+
     def _process_documents(
         self, documents, processors, langs, ask_before_saving=True, excluded_ids=None
     ):
@@ -692,7 +730,9 @@ def get_modified_documents(
 
         return result
 
-    def fix_recent_changes(self, oldest_date, newest_date, lang, ask_before_saving):
+    def clean_recent_changes(self, days, lang, ask_before_saving):
+        newest_date = utils.today().replace(hour=0, minute=0, second=0, microsecond=0)
+        oldest_date = newest_date - timedelta(days=days)
 
         excluded_ids = [
             996571,

campbot/objects.py

@@ -67,6 +67,7 @@ def __getattr__(self, item):
             )
 
         if item not in self:  # pragma: no cover
+            print("666777", self)
             raise AttributeError(
                 "Object {} has not attribute {}".format(self.__class__.__name__, item)
             )
@@ -83,11 +84,11 @@ def _convert_list(self, name, constructor):
         if name in self:
             self[name] = [constructor(self._campbot, data) for data in self[name]]
 
-    def _convert_dict(self, name, constructor):
-        if name in self:
-            self[name] = {
-                key: constructor(self._campbot, self[name][key]) for key in self[name]
-            }
+    # def _convert_dict(self, name, constructor):
+    #     if name in self:
+    #         self[name] = {
+    #             key: constructor(self._campbot, self[name][key]) for key in self[name]
+    #         }
 
 
 class Version(BotObject):
@@ -439,21 +440,3 @@ def __init__(self, campbot, data):
 class Poll(BotObject):
     def __init__(self, campbot, data):
         super(Poll, self).__init__(campbot, data)
-        self._convert_list("options", PollOption)
-
-
-class PollOption(BotObject):
-    def get_voters(self, post_id, poll_name):
-        url = "/polls/voters.json?post_id={}&poll_name={}&option_id={}&offset={}"
-        offset = 0
-        while True:
-            data = self._campbot.forum.get(
-                url.format(post_id, poll_name, self.id, offset)
-            )[poll_name]
-
-            if self.id not in data or len(data[self.id]) == 0:
-                return
-            for voter in data[self.id]:
-                yield ForumUser(self._campbot, voter)
-
-            offset += 1

docs/API/CampBot.rst

@@ -2,7 +2,7 @@ CampBot class
 =============
 
 .. autoclass:: campbot.CampBot
-   :members: login, clean, export, export_contributions, fix_recent_changes, get_documents,  get_users_from_route
+   :members: login, clean, export, export_contributions, clean_recent_changes, get_documents,  get_users_from_route
 
 Properties
 ----------

docs/CLI/index.rst

@@ -6,14 +6,15 @@ Here isthe help you get by typing ``campbot`` in you command line tool :
 .. code-block:: bash
 
     CampBot, Python bot framework for camptocamp.org
-    
+
     Usage:
-      campbot check_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
+      campbot clean_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
+      campbot report_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch]
       campbot clean <url_or_file> <langs> [--login=<login>] [--password=<password>] [--delay=<seconds>] [--batch] [--bbcode]
       campbot contribs [--out=<filename>] [--starts=<start_date>] [--ends=<end_date>] [--delay=<seconds>]
       campbot export <url> [--out=<filename>] [--delay=<seconds>]
-    
-    
+
+
     Options:
       --login=<login>           Bot login
       --password=<password>     Bot password
@@ -22,10 +23,11 @@ Here isthe help you get by typing ``campbot`` in you command line tool :
       --delay=<seconds>         Minimum delay between each request. Default : 3 seconds
       --bbcode                  Clean old BBCode in markdown
       --out=<filename>          Output file name. Default value will depend on process
-    
-    
+
+
     Commands:
-      check_rc      Check (and clean) recent changes.
+      report_rc     Make quality report on recent changes.
+      clean_rc      Clean recent changes.
       clean         Clean documents.
                     <url_or_file> is like https://www.camptocamp.org/routes#a=523281, or, simplier, routes#a=523281. 
                     filename is also accepted, and must be like : 
@@ -43,4 +45,4 @@ Here isthe help you get by typing ``campbot`` in you command line tool :
    export
    contribs
    clean
-   check_rc
+   report_rc

docs/CLI/check_rc.rst → docs/CLI/report_rc.rst

@@ -1,10 +1,7 @@
-Check (and clean) recent changes
+Quality report on recent changes
 ================================
 
-This procedure will execute two process : 
-
-1. All corrections made by :doc:`clean </CLI/clean>` procedure on every documents recently modified (bot modifications not included)
-2. Check that last day modifications pass these tests 
+This procedure will check that last day modifications pass these tests 
 
    * History is filled
    * No big deletions
@@ -19,7 +16,7 @@ Command line
 
 .. code-block:: bash
 
-    campbot check_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>]
+    campbot report_rc <days> [--login=<login>] [--password=<password>] [--delay=<seconds>]
 
 Arguments and options
 ---------------------
@@ -31,7 +28,7 @@ Arguments and options
 
 .. warning::
 
-    This process is quite long, especially for the second part. So, please consider this two points :
+    This process is quite long. So, please consider this two points :
 
     * Execute it when camptocamp is not overloaded
     * and use a delay of 0.1 seconds. 

tests/data/route.json

@@ -26,7 +26,7 @@
       "lang": "fr",
       "title_prefix": "Pedraforca - Dent dels Cabirols",
       "version": 8,
-      "route_history": "Ouverture : 7 juillet 1968 - R. Albert, J. Cerd\u00e0, L. Costa, A. Francesc.",
+      "route_history": null,
       "summary": null,
       "external_resources": "- \"**Pedraforca todas las ascensiones y escaladas**\", Joan Miquel Dalmau \n- [Sch\u00e9ma](http://www.lanochedelloro.es/wp-content/uploads/2017/03/7dentcabCivis.png) sur [La Noche del Loro]( http://www.lanochedelloro.es/index.php/pedraforca/).\n- [Photo]( http://2.bp.blogspot.com/-rN3ygAFu_iI/TwHHkTywKHI/AAAAAAAAADM/FkHHrjX3FaY/s1600/Vies+Cabirols.jpg) sur [Refuge Llu\u00eds Estasen](http://refugiestasen.blogspot.fr/2012/01/escalades-al-pedraforca.html) (2012).\n- Topo, photos sur [Escalatroncs](https://escalatroncs.com/2017/09/17/cerda-albert-same-cabirols-pedraforca/).\n"
     }

tests/fixtures.py

@@ -4,9 +4,14 @@
 import os
 
 
-def get_message(filename):
+def get_message(filename, overwrite_properties=None):
     filename = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data/" + filename + ".json")
-    return json.load(io.open(filename, encoding="utf-8"))
+    data = json.load(io.open(filename, encoding="utf-8"))
+
+    if overwrite_properties:
+        data.update(overwrite_properties)
+
+    return data
 
 
 def _wiki(method, url, answer):
@@ -30,7 +35,7 @@ def _forum(method, url, answer):
     _wiki('GET', r'waypoints/\d+/../\d+', get_message("waypoint_version")),
     _wiki('GET', r'routes/293549/fr/(1738922|1738528)', get_message("route_version")),
     _wiki('GET', r'routes/293549/fr/(880880|978249|478470|1738923)', get_message("route_version2")),
-    _wiki('GET', r'routes/952126', {'protected': True, 'document_id': 952126}),
+    _wiki('GET', r'routes/952126', get_message("route", {'protected': True, 'document_id': 952126})),
     _wiki('GET', r'routes/952167', {'redirects_to': 952126}),
     _wiki('GET', r'profiles/3199', {'document_id': 3199}),
     _wiki('GET', r'documents/changes\?.*&u=3199.*', {'feed': []}),
@@ -78,10 +83,10 @@ def _forum(method, url, answer):
     # Misc
     _wiki('GET', r'search.*', get_message("search_user")),
     _forum('POST', r'posts', {}),
-    _forum('GET', r'polls/voters.json.*&offset=0', {"poll": {"option_id": [{"username": "CharlesB"},
+    _forum('GET', r'polls/voters.json.*&page=1', {"voters": {"option_id": [{"username": "CharlesB"},
                                                                            {"username": "grimpeur8b"},
                                                                            {"username": "charlesdet"}]}}),
-    _forum('GET', r'polls/voters.json.*&offset=1', {"poll": {}}),
+    _forum('GET', r'polls/voters.json.*&page=2', {"voters": {"option_id": []}}),
 
     ('PUT', '.*', {}),
 ]

tests/test_miscs.py

@@ -53,7 +53,9 @@ def test_main_entry_point(fix_requests, ids_files):
 
     main(get_main_args("contribs"))
     main(get_main_args("export"))
-    main(get_main_args("check_rc"))
+    main(get_main_args("report_rc"))
+    main(get_main_args("report", {"<url_or_file>": "routes#w=123"}))
+    main(get_main_args("clean_rc"))
     main(get_main_args("clean", {"<url_or_file>": "routes#w=123"}))
     main(get_main_args("clean", {"<url_or_file>": "waypoints#w=123"}))
     main(get_main_args("clean", {"<url_or_file>": ids_files}))
@@ -210,17 +212,20 @@ def test_misc():
 def get_main_args(action, others=None):
     # noinspection PyDictCreation
     result = {
-        "check_rc": False,
+        "report_rc": False,
+        "clean_rc": False,
         "contribs": False,
         "export": False,
         "clean": False,
+        "report": False,
         "--delay": 0.01,
         "--login": "x",
         "--password": "y",
         "--lang": "fr",
         "--bbcode": True,
         "--batch": True,
         "<days>": "1",
+        "<lang>": "fr",
         "<langs>": "fr,de",
         "<url>": "outings#u=286726",
         "<url_or_file>": "outings#u=286726",
@@ -327,3 +332,11 @@ def test_get_closest_documents(fix_requests):
     bot = CampBot()
 
     bot.find_closest_documents(objects.Waypoint, 289284, 6175526, 2000)
+
+
+def test_get_voters(fix_requests):
+    from campbot import CampBot, objects
+
+    bot = CampBot()
+
+    bot.forum.get_voters(1234, "poll", "option_id")