Remove grab.util.misc module. Change way of tracking error names in s…

…tat module.
lorien · Dec 21, 2022 · 929f1cc · 929f1cc
1 parent c4ce649
commit 929f1cc
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 52 deletions.
diff --git a/grab/spider/base.py b/grab/spider/base.py
@@ -35,7 +35,6 @@
 from grab.spider.task import Task
 from grab.types import GrabConfig
 from grab.util.metrics import format_traffic_value
-from grab.util.misc import camel_case_to_underscore
 
 from .interface import FatalErrorQueueItem
 from .service.network import NetworkResult
@@ -580,7 +579,17 @@ def shutdown_services(self, services: list[BaseService]) -> None:
         logger.debug("Work done")
 
     def log_failed_network_result(self, res: NetworkResult) -> None:
-        msg = ("http-%s" % res["grab"].doc.code) if res["ok"] else res["error_abbr"]
+        orig_exc = (
+            res["exc"].original_exc
+            if isinstance(res["exc"], OriginalExceptionGrabError)
+            else res["exc"]
+        )
+        msg = (
+            ("http-%s" % res["grab"].doc.code)
+            if res["ok"]
+            else orig_exc.__class__.__name__
+        )
+
         self.stat.inc("error:%s" % msg)
 
     def log_rejected_task(self, task: Task, reason: str) -> None:
@@ -622,7 +631,7 @@ def srv_process_service_result(
         * ResponseNotValid-based exception
         * Arbitrary exception
         * Network response:
-            {ok, ecode, emsg, error_abbr, exc, grab, grab_config_backup}
+            {ok, ecode, emsg, exc, grab, grab_config_backup}
 
         Exception can come only from parser_service and it always has
         meta {"from": "parser", "exc_info": <...>}
@@ -690,13 +699,11 @@ def srv_process_task(self, task: Task) -> None:
             self.stat.inc("spider:request-network")
             self.stat.inc("spider:task-%s-network" % task.name)
 
-            # self.freelist.pop()
             try:
                 result: dict[str, Any] = {
                     "ok": True,
                     "ecode": None,
                     "emsg": None,
-                    "error_abbr": None,
                     "grab": grab,
                     "grab_config_backup": (grab_config_backup),
                     "task": task,
@@ -710,47 +717,15 @@ def srv_process_task(self, task: Task) -> None:
                     GrabInvalidResponse,
                     GrabTooManyRedirectsError,
                 ) as ex:
-                    is_redir_err = isinstance(ex, GrabTooManyRedirectsError)
-                    orig_exc_name = (
-                        ex.original_exc.__class__.__name__
-                        if hasattr(ex, "original_exc")
-                        else None
-                    )
-                    # UnicodeError: see #323
-                    ex_cls = (
-                        ex
-                        if (
-                            not isinstance(ex, OriginalExceptionGrabError)
-                            or isinstance(ex, GrabInvalidUrl)
-                            or orig_exc_name == "error"
-                            or orig_exc_name == "UnicodeError"
-                        )
-                        else cast(OriginalExceptionGrabError, ex).original_exc
-                    )
-                    result.update(
-                        {
-                            "ok": False,
-                            "exc": ex,
-                            "error_abbr": (
-                                "too-many-redirects"
-                                if is_redir_err
-                                else self.make_class_abbr(ex_cls.__class__.__name__)
-                            ),
-                        }
-                    )
+                    result.update({"ok": False, "exc": ex})
                 self.task_dispatcher.input_queue.put((result, task, None))
             finally:
                 pass
-                # self.freelist.append(1)
         else:
             self.log_rejected_task(task, reason)
             handler = self.get_fallback_handler(task)
             if handler:
                 handler(task)
 
-    def make_class_abbr(self, name: str) -> str:
-        val = camel_case_to_underscore(name)
-        return val.replace("_", "-")
-
 
 # pylint: enable=too-many-instance-attributes, too-many-public-methods
diff --git a/grab/util/misc.py b/grab/util/misc.py
diff --git a/tests/test_spider_error.py b/tests/test_spider_error.py
@@ -1,3 +1,5 @@
+from pprint import pprint  # pylint: disable=unused-import
+
 from test_server import Response
 
 from grab import Grab, GrabTimeoutError
@@ -118,4 +120,4 @@ def task_page(self, grab, unused_task):
 
         bot = build_spider(SimpleSpider)
         bot.run()
-        self.assertTrue("error:read-timeout-error" in bot.stat.counters)
+        self.assertTrue("error:ReadTimeoutError" in bot.stat.counters)
diff --git a/tests/test_spider_redirect.py b/tests/test_spider_redirect.py
@@ -29,7 +29,7 @@ def task_page(self, unused_grab, unused_task):
         bot.run()
 
         self.assertEqual(1, len(bot.runtime_events["network-count-rejected"]))
-        self.assertTrue("error:too-many-redirects" in bot.stat.counters)
+        self.assertTrue("error:GrabTooManyRedirectsError" in bot.stat.counters)
 
     def test_redirect_with_invalid_byte(self):
         server = self.server
@@ -57,7 +57,7 @@ def task_page(self, unused_grab, unused_task):
         self.assertEqual(1, len(bot.runtime_events["network-count-rejected"]))
         self.assertTrue(
             # bot.stat.counters["error:new-connection-error"] == 5
-            bot.stat.counters["error:location-parse-error"]
+            bot.stat.counters["error:LocationParseError"]
             # or bot.stat.counters["error:grab-could-not-resolve-host-error"] == 5
             # or bot.stat.counters["error:couldnt-resolve-host"] == 5
         )