Merge 6759a29 into 3329506

remotecv/celery_tasks.py

@@ -1,6 +1,8 @@
 from celery import Celery
 
 from remotecv.pyres_tasks import DetectTask
+from remotecv.timing import get_time, get_interval
+from remotecv.utils import context
 
 
 class CeleryTasks:
@@ -19,8 +21,15 @@ def __init__(self, key_id, key_secret, region, timeout=None, polling_interval=No
     def get_detect_task(self):
         @self.celery.task(ignore_result=True, acks_late=True)
         def detect_task(detection_type, image_path, key):
+            start_time = get_time()
             DetectTask.perform(detection_type, image_path, key)
 
+            context.metrics.timing(
+                "worker.celery_task.time",
+                get_interval(start_time, get_time()),
+            )
+            context.metrics.incr("worker.celery_task.total")
+
         return detect_task
 
     def run_commands(self, args, log_level=None):

remotecv/http_loader.py

@@ -1,9 +1,30 @@
 import re
+
 from urllib.request import unquote, urlopen
+from urllib.parse import urlparse
+from remotecv.timing import get_time, get_interval
+from remotecv.utils import context
 
 
 def load_sync(path):
+    start_time = get_time()
     if not re.match(r"^https?", path):
         path = f"http://{path}"
     path = unquote(path)
-    return urlopen(path).read()
+    netloc = urlparse(path).netloc.replace(".", "_")
+    response = urlopen(path)
+    code = response.code
+    result = response.read()
+
+    context.metrics.incr("worker.original_image.response_bytes", len(result))
+    context.metrics.timing(
+        f"worker.original_image.fetch.{code}.{netloc}",
+        get_interval(start_time, get_time()),
+    )
+    context.metrics.incr(
+        f"worker.original_image.fetch.{code}.{netloc}",
+    )
+    context.metrics.incr(f"worker.original_image.status.{code}")
+    context.metrics.incr(f"worker.original_image.status.{code}.{netloc}")
+
+    return result

remotecv/image_processor.py

@@ -7,6 +7,8 @@
 from remotecv.detectors.glasses_detector import GlassesDetector
 from remotecv.detectors.profile_detector import ProfileDetector
 from remotecv.image import Image
+from remotecv.timing import get_time, get_interval
+from remotecv.utils import context
 
 
 class ImageProcessor:
@@ -27,7 +29,16 @@ def detect(self, detector, image_data):
 
         for detector_key in detector.split("+"):
             try:
-                result = result + self.detectors[detector_key].detect(image)
+                start_time = get_time()
+                points = self.detectors[detector_key].detect(image)
+                context.metrics.timing(
+                    f"worker.{detector_key}.time",
+                    get_interval(start_time, get_time()),
+                )
+                if points:
+                    context.metrics.incr(f"worker.{detector_key}.detected")
+
+                result = result + points
             except KeyError as key_error:
                 raise AttributeError("Detector unavailable") from key_error
         return result

remotecv/importer.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2022, globo.com <thumbor@g.globo>
+
+from importlib import import_module
+
+
+class Importer:
+    @classmethod
+    def import_class(cls, class_name, conf_value):
+        module = import_module(conf_value)
+        return getattr(module, class_name)

remotecv/metrics/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2022, globo.com <thumbor@g.globo>
+
+
+class BaseMetrics:
+    def __init__(self, config):
+        self.config = config
+
+    def initialize(self):
+        pass
+
+    def incr(self, metricname, value=1):
+        raise NotImplementedError()
+
+    def timing(self, metricname, value):
+        raise NotImplementedError()

remotecv/metrics/logger_metrics.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2022, globo.com <thumbor@g.globo>
+
+from remotecv.metrics import BaseMetrics
+from remotecv.utils import logger
+
+
+class Metrics(BaseMetrics):
+    def incr(self, metricname, value=1):
+        logger.debug("METRICS: inc: %s:%d", metricname, value)
+
+    def timing(self, metricname, value):
+        logger.debug("METRICS: timing: %s:%d", metricname, value)

remotecv/pyres_tasks.py

@@ -1,5 +1,6 @@
 from remotecv.image_processor import ImageProcessor
-from remotecv.utils import config, logger
+from remotecv.timing import get_time, get_interval
+from remotecv.utils import config, logger, context
 
 # pylint: disable=no-member
 
@@ -10,8 +11,15 @@ class DetectTask:
 
     @classmethod
     def perform(cls, detection_type, image_path, key):
+        start_time = get_time()
         logger.info("Detecting %s for %s", detection_type, image_path)
         image_data = config.loader.load_sync(image_path)
         points = cls.processor.detect(detection_type, image_data)
         result_store = config.store.ResultStore(config)
         result_store.store(key, points)
+
+        context.metrics.timing(
+            "worker.pyres_task.time",
+            get_interval(start_time, get_time()),
+        )
+        context.metrics.incr("worker.pyres_task.total")

remotecv/timing.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+
+# thumbor imaging service
+# https://github.com/thumbor/thumbor/wiki
+
+# Licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license
+# Copyright (c) 2022, globo.com <thumbor@g.globo>
+
+import time
+
+
+def get_time():
+    return time.perf_counter_ns()
+
+
+def get_interval(start, end):
+    return nano_to_ms(end - start)
+
+
+def nano_to_ms(ns_time):
+    return ns_time / 1e6

remotecv/utils.py

@@ -17,8 +17,13 @@ class Config:
     pass
 
 
+class Context:
+    pass
+
+
 logger = logging.getLogger("remotecv")  # pylint: disable=invalid-name
 config = Config()  # pylint: disable=invalid-name
+context = Context()
 
 SINGLE_NODE = "single_node"
 SENTINEL = "sentinel"

remotecv/worker.py

@@ -16,8 +16,9 @@
 from threading import Thread
 
 from remotecv.error_handler import ErrorHandler
-from remotecv.utils import config, redis_client, SINGLE_NODE, SENTINEL
 from remotecv.healthcheck import HealthCheckHandler
+from remotecv.importer import Importer
+from remotecv.utils import config, redis_client, SINGLE_NODE, SENTINEL, context
 
 
 def start_pyres_worker():
@@ -63,6 +64,12 @@ def serve_forever(httpd):
     thread.start()
 
 
+def import_modules():
+    Metrics = Importer.import_class("Metrics", config.metrics)
+    context.metrics = Metrics(config)
+    context.metrics.initialize()
+
+
 def main(params=None):
     if params is None:
         params = sys.argv[1:]
@@ -150,6 +157,11 @@ def main(params=None):
     other_group.add_argument(
         "--sentry_url", default=None, help="URL used to send errors to sentry"
     )
+    other_group.add_argument(
+        "--metrics",
+        default="remotecv.metrics.logger_metrics",
+        help="Metrics client, should be the full name of a python module",
+    )
 
     memcache_store_group = parser.add_argument_group("Memcache store arguments")
     memcache_store_group.add_argument(
@@ -186,13 +198,16 @@ def main(params=None):
     config.log_level = arguments.level.upper()
     config.loader = import_module(arguments.loader)
     config.store = import_module(arguments.store)
+    config.metrics = arguments.metrics
 
     config.memcache_hosts = arguments.memcache_hosts
 
     config.extra_args = sys.argv[:1] + arguments.args
 
     config.error_handler = ErrorHandler(arguments.sentry_url)
 
+    import_modules()
+
     if arguments.with_healthcheck:
         start_http_server()
 

setup.py

@@ -70,6 +70,7 @@
         "remotecv.detectors.feature_detector",
         "remotecv.detectors.glasses_detector",
         "remotecv.detectors.profile_detector",
+        "remotecv.metrics",
         "remotecv.result_store",
     ],
     package_dir={"": "."},

tests/test_complete_detector.py

@@ -1,16 +1,18 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.detectors.complete_detector import CompleteDetector
+from remotecv.utils import context
 from tests import create_image
 
 
 class CompleteDetectorTestCase(TestCase):
     def test_should_detect_something(self):
+        context.metrics = mock.Mock()
         detection_result = CompleteDetector().detect(create_image("profile_face.jpg"))
         expect(len(detection_result)).to_be_greater_than(20)
         expect(detection_result[0][0]).to_be_numeric()

tests/test_face_detector.py

@@ -1,15 +1,19 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.detectors.face_detector import FaceDetector
+from remotecv.utils import context
 from tests import create_image
 
 
 class FaceDetectorTestCase(TestCase):
+    def setUp(self):
+        context.metrics = mock.Mock()
+
     def test_should_detect_one_face(self):
         detection_result = FaceDetector().detect(create_image("one_face.jpg"))
         expect(detection_result).to_length(1)

tests/test_feature_detector.py

@@ -1,15 +1,19 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.detectors.feature_detector import FeatureDetector
+from remotecv.utils import context
 from tests import create_image
 
 
 class FeatureDetectorTestCase(TestCase):
+    def setUp(self):
+        context.metrics = mock.Mock()
+
     def test_should_detect_multiple_points(self):
         detection_result = FeatureDetector().detect(create_image("no_face.jpg"))
         expect(len(detection_result)).to_be_greater_than(4)

tests/test_glasses_detector.py

@@ -1,15 +1,19 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.detectors.glasses_detector import GlassesDetector
+from remotecv.utils import context
 from tests import create_image
 
 
 class GlassesDetectorTestCase(TestCase):
+    def setUp(self):
+        context.metrics = mock.Mock()
+
     def test_should_detect_glasses(self):
         detection_result = GlassesDetector().detect(create_image("glasses.jpg"))
         expect(detection_result).to_length(2)

tests/test_image_processor.py

@@ -1,12 +1,16 @@
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.image_processor import ImageProcessor
+from remotecv.utils import context
 from tests import read_fixture
 
 
 class ImageProcessorTest(TestCase):
+    def setUp(self):
+        context.metrics = mock.Mock()
+
     def test_when_detector_unavailable(self):
         image_processor = ImageProcessor()
         with expect.error_to_happen(AttributeError):

tests/test_profile_detector.py

@@ -1,15 +1,19 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from preggy import expect
 
 from remotecv.detectors.profile_detector import ProfileDetector
+from remotecv.utils import context
 from tests import create_image
 
 
 class ProfileDetectorTestCase(TestCase):
+    def setUp(self):
+        context.metrics = mock.Mock()
+
     def test_should_detect_one_face(self):
         detection_result = ProfileDetector().detect(create_image("profile_face.jpg"))
         expect(detection_result).to_length(1)