Adds resize helper functions

Also, adds an ability to use run_classifier with debug = True. (debug mode)

Author: Alex-EEE

.gitignore

@@ -5,3 +5,5 @@ build
 *.jpg
 .DS_Store
 .venv/
+.vscode
+*.eim

README.md

@@ -27,6 +27,8 @@ This library lets you run machine learning models and collect sensor data on Lin
     ```
 
     For the computer vision examples you'll want `opencv-python>=4.5.1.48`
+    Note on macOS on apple silicon, you will need to use a later version,
+    4.10.0.84 tested and installs cleanly
 
 ## Collecting data
 

edge_impulse_linux/audio.py

@@ -130,8 +130,8 @@ def __init__(self, model_path: str):
         self.window_size = 0
         self.labels = []
 
-    def init(self):
-        model_info = super(AudioImpulseRunner, self).init()
+    def init(self, debug=False):
+        model_info = super(AudioImpulseRunner, self).init(debug)
         if model_info['model_parameters']['frequency'] == 0:
             raise Exception('Model file "' + self._model_path + '" is not suitable for audio recognition')
 

edge_impulse_linux/image.py

@@ -14,18 +14,20 @@ def __init__(self, model_path: str):
         self.dim = (0, 0)
         self.videoCapture = cv2.VideoCapture()
         self.isGrayscale = False
+        self.resizeMode = ''
 
-    def init(self):
-        model_info = super(ImageImpulseRunner, self).init()
-        width = model_info['model_parameters']['image_input_width'];
-        height = model_info['model_parameters']['image_input_height'];
+    def init(self, debug=False):
+        model_info = super(ImageImpulseRunner, self).init(debug)
+        width = model_info['model_parameters']['image_input_width']
+        height = model_info['model_parameters']['image_input_height']
 
         if width == 0 or height == 0:
             raise Exception('Model file "' + self._model_path + '" is not suitable for image recognition')
 
         self.dim = (width, height)
         self.labels = model_info['model_parameters']['labels']
-        self.isGrayscale =  model_info['model_parameters']['image_channel_count'] == 1
+        self.isGrayscale = model_info['model_parameters']['image_channel_count'] == 1
+        self.resizeMode = model_info['model_parameters'].get('image_resize_mode', 'not-reported')
         return model_info
 
     def __enter__(self):
@@ -69,7 +71,7 @@ def classifier(self, videoDeviceId = 0):
                 res = self.classify(features)
                 yield res, cropped
 
-    # This expects images in RGB format (not BGR)
+    # This expects images in RGB format (not BGR), DEPRECATED, use get_features_from_image_auto_studio_setings
     def get_features_from_image(self, img, crop_direction_x='center', crop_direction_y='center'):
         features = []
 
@@ -129,3 +131,107 @@ def get_features_from_image(self, img, crop_direction_x='center', crop_direction
                 features.append((r << 16) + (g << 8) + b)
 
         return features, cropped
+
+    def get_features_from_image_auto_studio_setings(self, img):
+        if self.resizeMode == '':
+            raise Exception(
+                'Runner has not initialized, please call init() first')
+        if self.resizeMode == 'not-reported':
+            raise Exception(
+                'Model file "' + self._model_path + '" does not report the image resize mode\n'
+                'Please update the model file via edge-impulse-linux-runner --download')
+        return get_features_from_image_with_studio_mode(img, self.resizeMode, self.dim[0], self.dim[1], self.isGrayscale)
+
+
+def resize_with_letterbox(image, target_width, target_height):
+    """Resize an image while maintaining aspect ratio using letterboxing.
+
+    Args:
+        image: The input image as a NumPy array.
+        target_size: A tuple (width, height) specifying the desired output size.
+
+    Returns:
+        The resized image as a NumPy array and the letterbox dimensions.
+    """
+
+    height, width = image.shape[:2]
+
+    # Calculate scale factors to preserve aspect ratio
+    scale_x = target_width / width
+    scale_y = target_height / height
+    scale = min(scale_x, scale_y)
+
+    # Calculate new dimensions and padding
+    new_width = int(width * scale)
+    new_height = int(height * scale)
+    top_pad = (target_height - new_height) // 2
+    bottom_pad = target_height - new_height - top_pad
+    left_pad = (target_width - new_width) // 2
+    right_pad = target_width - new_width - left_pad
+
+    # Resize image and add padding
+    resized_image = cv2.resize(image, (new_width, new_height))
+    padded_image = cv2.copyMakeBorder(resized_image, top_pad, bottom_pad, left_pad, right_pad, cv2.BORDER_CONSTANT, value=0)
+
+    return padded_image
+
+
+def get_features_from_image_with_studio_mode(img, mode, output_width, output_height, is_grayscale):
+    """
+    Extract features from an image using different resizing modes suitable for Edge Impulse Studio.
+
+    Args:
+        img (numpy.ndarray): The input image as a NumPy array.
+        mode (str): The resizing mode to use. Options are 'fit-shortest', 'fit-longest', and 'squash'.
+        output_width (int): The desired output width of the image.
+        output_height (int): The desired output height of the image.
+        is_grayscale (bool): Whether the output image should be converted to grayscale.
+
+    Returns:
+        tuple: A tuple containing:
+            - features (list): A list of pixel values in the format (R << 16) + (G << 8) + B for color images,
+              or (P << 16) + (P << 8) + P for grayscale images.
+            - resized_img (numpy.ndarray): The resized image as a NumPy array.
+    """
+    features = []
+
+    in_frame_cols = img.shape[1]
+    in_frame_rows = img.shape[0]
+
+    if mode == 'fit-shortest':
+        aspect_ratio = output_width / output_height
+        if in_frame_cols / in_frame_rows > aspect_ratio:
+            # Image is wider than target aspect ratio
+            new_width = int(in_frame_rows * aspect_ratio)
+            offset = (in_frame_cols - new_width) // 2
+            cropped_img = img[:, offset:offset + new_width]
+        else:
+            # Image is taller than target aspect ratio
+            new_height = int(in_frame_cols / aspect_ratio)
+            offset = (in_frame_rows - new_height) // 2
+            cropped_img = img[offset:offset + new_height, :]
+
+        resized_img = cv2.resize(cropped_img, (output_width, output_height), interpolation=cv2.INTER_AREA)
+    elif mode == 'fit-longest':
+        resized_img = resize_with_letterbox(img, output_width, output_height)
+    elif mode == 'squash':
+        resized_img = cv2.resize(img, (output_width, output_height), interpolation=cv2.INTER_AREA)
+    else:
+        raise ValueError(f"Unsupported mode: {mode}")
+
+    if is_grayscale:
+        resized_img = cv2.cvtColor(resized_img, cv2.COLOR_BGR2GRAY)
+        pixels = np.array(resized_img).flatten().tolist()
+
+        for p in pixels:
+            features.append((p << 16) + (p << 8) + p)
+    else:
+        pixels = np.array(resized_img).flatten().tolist()
+
+        for ix in range(0, len(pixels), 3):
+            r = pixels[ix + 0]
+            g = pixels[ix + 1]
+            b = pixels[ix + 2]
+            features.append((r << 16) + (g << 8) + b)
+
+    return features, resized_img

edge_impulse_linux/runner.py

@@ -7,69 +7,83 @@
 import socket
 import json
 
+
 def now():
     return round(time.time() * 1000)
 
+
 class ImpulseRunner:
     def __init__(self, model_path: str):
         self._model_path = model_path
         self._tempdir = None
         self._runner = None
         self._client = None
         self._ix = 0
+        self._debug = False
 
-    def init(self):
-        if (not os.path.exists(self._model_path)):
-            raise Exception('Model file does not exist: ' + self._model_path)
+    def init(self, debug=False):
+        if not os.path.exists(self._model_path):
+            raise Exception("Model file does not exist: " + self._model_path)
 
-        if (not os.access(self._model_path, os.X_OK)):
+        if not os.access(self._model_path, os.X_OK):
             raise Exception('Model file "' + self._model_path + '" is not executable')
 
+        self._debug = debug
         self._tempdir = tempfile.mkdtemp()
-        socket_path = os.path.join(self._tempdir, 'runner.sock')
-        self._runner = subprocess.Popen([self._model_path, socket_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        while not os.path.exists(socket_path) or not self._runner.poll() is None:
+        socket_path = os.path.join(self._tempdir, "runner.sock")
+        cmd = [self._model_path, socket_path]
+        if debug:
+            self._runner = subprocess.Popen(cmd)
+        else:
+            self._runner = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+
+        while not os.path.exists(socket_path) or self._runner.poll() is not None:
             time.sleep(0.1)
 
-        if not self._runner.poll() is None:
-            raise Exception('Failed to start runner (' + str(self._runner.poll()) + ')')
+        if self._runner.poll() is not None:
+            raise Exception("Failed to start runner (" + str(self._runner.poll()) + ")")
 
         self._client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         self._client.connect(socket_path)
 
         return self.hello()
 
     def stop(self):
-        if (self._tempdir):
+        if self._tempdir:
             shutil.rmtree(self._tempdir)
 
-        if (self._client):
+        if self._client:
             self._client.close()
 
-        if (self._runner):
+        if self._runner:
             os.kill(self._runner.pid, signal.SIGINT)
             # todo: in Node we send a SIGHUP after 0.5sec if process has not died, can we do this somehow here too?
 
     def hello(self):
-        msg = { "hello": 1 }
+        msg = {"hello": 1}
         return self.send_msg(msg)
 
     def classify(self, data):
-        msg = { "classify": data }
+        msg = {"classify": data}
+        if self._debug:
+            msg["debug"] = True
         return self.send_msg(msg)
 
     def send_msg(self, msg):
         t_send_msg = now()
 
         if not self._client:
-            raise Exception('ImpulseRunner is not initialized')
+            raise Exception("ImpulseRunner is not initialized (call init())")
 
         self._ix = self._ix + 1
         ix = self._ix
 
         msg["id"] = ix
-        self._client.send(json.dumps(msg).encode('utf-8'))
+        self._client.send(json.dumps(msg).encode("utf-8"))
 
         t_sent_msg = now()
 
@@ -81,29 +95,29 @@ def send_msg(self, msg):
 
         braces_open = 0
         braces_closed = 0
-        line = ''
+        line = ""
         resp = None
 
-        for c in data.decode('utf-8'):
-            if c == '{':
+        for c in data.decode("utf-8"):
+            if c == "{":
                 line = line + c
                 braces_open = braces_open + 1
-            elif c == '}':
+            elif c == "}":
                 line = line + c
                 braces_closed = braces_closed + 1
-                if (braces_closed == braces_open):
+                if braces_closed == braces_open:
                     resp = json.loads(line)
             elif braces_open > 0:
                 line = line + c
 
-            if (not resp is None):
+            if resp is not None:
                 break
 
-        if (resp is None):
-            raise Exception('No data or corrupted data received')
+        if resp is None:
+            raise Exception("No data or corrupted data received")
 
-        if (resp["id"] != ix):
-            raise Exception('Wrong id, expected: ' + str(ix) + ' but got ' + resp["id"])
+        if resp["id"] != ix:
+            raise Exception("Wrong id, expected: " + str(ix) + " but got " + resp["id"])
 
         if not resp["success"]:
             raise Exception(resp["error"])

examples/audio/classify.py

@@ -41,6 +41,7 @@ def main(argv):
     with AudioImpulseRunner(modelfile) as runner:
         try:
             model_info = runner.init()
+            # model_info = runner.init(debug=True) # to get debug print out
             labels = model_info['model_parameters']['labels']
             print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
 

examples/custom/classify.py

@@ -54,6 +54,8 @@ def main(argv):
     runner = ImpulseRunner(modelfile)
     try:
         model_info = runner.init()
+        # model_info = runner.init(debug=True) # to get debug print out
+
         print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
 
         res = runner.classify(features)

examples/image/classify-full-frame.py

@@ -67,6 +67,8 @@ def main(argv):
     with ImageImpulseRunner(modelfile) as runner:
         try:
             model_info = runner.init()
+            # model_info = runner.init(debug=True) # to get debug print out
+
             print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
             labels = model_info['model_parameters']['labels']
             if len(args)>= 2:

examples/image/classify-image.py

@@ -39,6 +39,8 @@ def main(argv):
     with ImageImpulseRunner(modelfile) as runner:
         try:
             model_info = runner.init()
+            # model_info = runner.init(debug=True) # to get debug print out
+
             print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
             labels = model_info['model_parameters']['labels']
 
@@ -51,7 +53,10 @@ def main(argv):
             img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 
             # get_features_from_image also takes a crop direction arguments in case you don't have square images
-            features, cropped = runner.get_features_from_image(img)
+            # features, cropped = runner.get_features_from_image(img)
+
+            # this mode uses the same settings used in studio to crop and resize the input
+            features, cropped = runner.get_features_from_image_auto_studio_setings(img)
 
             res = runner.classify(features)
 

examples/image/classify-video.py

@@ -45,6 +45,7 @@ def main(argv):
     with ImageImpulseRunner(modelfile) as runner:
         try:
             model_info = runner.init()
+            # model_info = runner.init(debug=True) # to get debug print out
             print('Loaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
             labels = model_info['model_parameters']['labels']