vision-framework: initial release

plugins/tensorflow-lite/src/predict/__init__.py

@@ -27,7 +27,7 @@ async def to_thread(f):
     return await loop.run_in_executor(toThreadExecutor, f)
 
 async def ensureRGBData(data: bytes, size: Tuple[int, int], format: str):
-    if format != 'rgba':
+    if format == 'rgb':
         return Image.frombuffer('RGB', size, data)
 
     def convert():
@@ -38,6 +38,19 @@ def convert():
             rgba.close()
     return await to_thread(convert)
 
+async def ensureRGBAData(data: bytes, size: Tuple[int, int], format: str):
+    if format == 'rgba':
+        return Image.frombuffer('RGBA', size, data)
+
+    # this path should never be possible as all the image sources should be capable of rgba.
+    def convert():
+        rgb = Image.frombuffer('RGB', size, data)
+        try:
+            return rgb.convert('RGBA')
+        finally:
+            rgb.close()
+    return await to_thread(convert)
+
 def parse_label_contents(contents: str):
     lines = contents.splitlines()
     ret = {}
@@ -154,23 +167,36 @@ async def run_detection_image(self, image: scrypted_sdk.Image, detection_session
         iw, ih = image.width, image.height
         w, h = self.get_input_size()
 
-        resize = None
-        xs = w / iw
-        ys = h / ih
-        def cvss(point):
-            return point[0] / xs, point[1] / ys
-
-        if iw != w or ih != h:
-            resize = {
-                'width': w,
-                'height': h,
-            }
-
+        if w is None or h is None:
+            resize = None
+            w = image.width
+            h = image.height
+            def cvss(point):
+                return point
+        else:
+            resize = None
+            xs = w / iw
+            ys = h / ih
+            def cvss(point):
+                return point[0] / xs, point[1] / ys
+
+            if iw != w or ih != h:
+                resize = {
+                    'width': w,
+                    'height': h,
+                }
+
+        format = image.format or self.get_input_format()
         b = await image.toBuffer({
             'resize': resize,
-            'format': image.format or 'rgb',
+            'format': format,
         })
-        data = await ensureRGBData(b, (w, h), image.format)
+
+        if self.get_input_format() == 'rgb':
+            data = await ensureRGBData(b, (w, h), format)
+        elif self.get_input_format() == 'rgba':
+            data = await ensureRGBAData(b, (w, h), format)
+
         try:
             ret = await self.safe_detect_once(data, settings, (iw, ih), cvss)
             return ret

plugins/vision-framework/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+out/
+node_modules/
+dist/
+.venv
+all_models*

plugins/vision-framework/.npmignore

@@ -0,0 +1,12 @@
+.DS_Store
+out/
+node_modules/
+*.map
+fs
+src
+.vscode
+dist/*.js
+dist/*.txt
+__pycache__
+all_models
+.venv

plugins/vision-framework/.vscode/launch.json

@@ -0,0 +1,30 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Scrypted Debugger",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "${config:scrypted.debugHost}",
+                "port": 10081
+            },
+            "justMyCode": false,
+            "preLaunchTask": "scrypted: deploy+debug",
+            "pathMappings": [
+                {
+                    "localRoot": "/Volumes/Dev/scrypted/server/python/",
+                    "remoteRoot": "/Volumes/Dev/scrypted/server/python/",
+                },
+                {
+                    "localRoot": "${workspaceFolder}/src",
+                    "remoteRoot": "${config:scrypted.pythonRemoteRoot}"
+                },
+
+            ]
+        }
+    ]
+}

plugins/vision-framework/.vscode/settings.json

@@ -0,0 +1,19 @@
+
+{
+    // docker installation
+    // "scrypted.debugHost": "koushik-thin",
+    // "scrypted.serverRoot": "/server",
+
+    // pi local installation
+    // "scrypted.debugHost": "192.168.2.119",
+    // "scrypted.serverRoot": "/home/pi/.scrypted",
+
+    // local checkout
+    "scrypted.debugHost": "127.0.0.1",
+    "scrypted.serverRoot": "/Users/koush/.scrypted",
+
+    "scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip",
+    "python.analysis.extraPaths": [
+        "./node_modules/@scrypted/sdk/types/scrypted_python"
+    ]
+}

plugins/vision-framework/.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "scrypted: deploy+debug",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "silent",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}",
+        },
+    ]
+}

plugins/vision-framework/README.md

@@ -0,0 +1,6 @@
+# Vision Framework Object Detection for Scrypted
+
+This plugin adds object detection capabilities to any camera in Scrypted. This plugin requires MacOS Vision Framework. Vision Framework utilizes the CPU, GPU, and Neural Cores (Apple Silicon).
+
+The Vision Framework Plugin should only be used if you are a Scrypted NVR user. It will provide no
+benefits to HomeKit, which does its own detection processing.

plugins/vision-framework/package.json

@@ -0,0 +1,44 @@
+{
+   "name": "@scrypted/vision-framework",
+   "description": "Scrypted Vision Framework Object Detection",
+   "keywords": [
+      "scrypted",
+      "plugin",
+      "vision framework",
+      "neural",
+      "object",
+      "detect",
+      "detection",
+      "people",
+      "person"
+   ],
+   "scripts": {
+      "scrypted-setup-project": "scrypted-setup-project",
+      "prescrypted-setup-project": "scrypted-package-json",
+      "build": "scrypted-webpack",
+      "prepublishOnly": "NODE_ENV=production scrypted-webpack",
+      "prescrypted-vscode-launch": "scrypted-webpack",
+      "scrypted-vscode-launch": "scrypted-deploy-debug",
+      "scrypted-deploy-debug": "scrypted-deploy-debug",
+      "scrypted-debug": "scrypted-debug",
+      "scrypted-deploy": "scrypted-deploy",
+      "scrypted-readme": "scrypted-readme",
+      "scrypted-package-json": "scrypted-package-json"
+   },
+   "scrypted": {
+      "name": "Vision Framework Object Detection",
+      "pluginDependencies": [
+         "@scrypted/objectdetector"
+      ],
+      "runtime": "python",
+      "type": "API",
+      "interfaces": [
+         "ObjectDetection",
+         "ObjectDetectionPreview"
+      ]
+   },
+   "devDependencies": {
+      "@scrypted/sdk": "file:../../sdk"
+   },
+   "version": "0.0.2"
+}

plugins/vision-framework/src/detect

@@ -0,0 +1 @@
+../../tensorflow-lite/src/detect

plugins/vision-framework/src/main.py

@@ -0,0 +1,4 @@
+from vision import VisionPlugin
+
+def create_scrypted_plugin():
+    return VisionPlugin()

plugins/vision-framework/src/predict

@@ -0,0 +1 @@
+../../tensorflow-lite/src/predict

plugins/vision-framework/src/requirements.txt

@@ -0,0 +1,2 @@
+pyobjc
+Pillow>=5.4.1