-
-
Notifications
You must be signed in to change notification settings - Fork 235
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
15 changed files
with
386 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
.DS_Store | ||
out/ | ||
node_modules/ | ||
dist/ | ||
.venv | ||
all_models* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
.DS_Store | ||
out/ | ||
node_modules/ | ||
*.map | ||
fs | ||
src | ||
.vscode | ||
dist/*.js | ||
dist/*.txt | ||
__pycache__ | ||
all_models | ||
sort_oh | ||
download_models.sh | ||
tsconfig.json | ||
.venv |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
{ | ||
// Use IntelliSense to learn about possible attributes. | ||
// Hover to view descriptions of existing attributes. | ||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 | ||
"version": "0.2.0", | ||
"configurations": [ | ||
{ | ||
"name": "Scrypted Debugger", | ||
"type": "python", | ||
"request": "attach", | ||
"connect": { | ||
"host": "${config:scrypted.debugHost}", | ||
"port": 10081 | ||
}, | ||
"justMyCode": false, | ||
"preLaunchTask": "scrypted: deploy+debug", | ||
"pathMappings": [ | ||
{ | ||
"localRoot": "/Volumes/Dev/scrypted/server/python/", | ||
"remoteRoot": "/Volumes/Dev/scrypted/server/python/", | ||
}, | ||
{ | ||
"localRoot": "${workspaceFolder}/src", | ||
"remoteRoot": "${config:scrypted.pythonRemoteRoot}" | ||
}, | ||
|
||
] | ||
} | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
|
||
{ | ||
// docker installation | ||
// "scrypted.debugHost": "koushik-ubuntu", | ||
// "scrypted.serverRoot": "/server", | ||
|
||
// pi local installation | ||
// "scrypted.debugHost": "192.168.2.119", | ||
// "scrypted.serverRoot": "/home/pi/.scrypted", | ||
|
||
// local checkout | ||
"scrypted.debugHost": "127.0.0.1", | ||
"scrypted.serverRoot": "/Users/koush/.scrypted", | ||
// "scrypted.debugHost": "koushik-winvm", | ||
// "scrypted.serverRoot": "C:\\Users\\koush\\.scrypted", | ||
|
||
"scrypted.pythonRemoteRoot": "${config:scrypted.serverRoot}/volume/plugin.zip", | ||
"python.analysis.extraPaths": [ | ||
"./node_modules/@scrypted/sdk/types/scrypted_python" | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
{ | ||
// See https://go.microsoft.com/fwlink/?LinkId=733558 | ||
// for the documentation about the tasks.json format | ||
"version": "2.0.0", | ||
"tasks": [ | ||
{ | ||
"label": "scrypted: deploy+debug", | ||
"type": "shell", | ||
"presentation": { | ||
"echo": true, | ||
"reveal": "silent", | ||
"focus": false, | ||
"panel": "shared", | ||
"showReuseMessage": true, | ||
"clear": false | ||
}, | ||
"command": "npm run scrypted-vscode-launch ${config:scrypted.debugHost}", | ||
}, | ||
] | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
# ONNX Object Detection for Scrypted | ||
|
||
This plugin adds object detection capabilities to any camera in Scrypted. Having a fast GPU and CPU is highly recommended. | ||
|
||
The ONNX Plugin should only be used if you are a Scrypted NVR user. It will provide no | ||
benefits to HomeKit, which does its own detection processing. |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
{ | ||
"name": "@scrypted/onnx", | ||
"description": "Scrypted ONNX Object Detection", | ||
"keywords": [ | ||
"scrypted", | ||
"plugin", | ||
"onnx", | ||
"motion", | ||
"object", | ||
"detect", | ||
"detection", | ||
"people", | ||
"person" | ||
], | ||
"scripts": { | ||
"scrypted-setup-project": "scrypted-setup-project", | ||
"prescrypted-setup-project": "scrypted-package-json", | ||
"build": "scrypted-webpack", | ||
"prepublishOnly": "NODE_ENV=production scrypted-webpack", | ||
"prescrypted-vscode-launch": "scrypted-webpack", | ||
"scrypted-vscode-launch": "scrypted-deploy-debug", | ||
"scrypted-deploy-debug": "scrypted-deploy-debug", | ||
"scrypted-debug": "scrypted-debug", | ||
"scrypted-deploy": "scrypted-deploy", | ||
"scrypted-readme": "scrypted-readme", | ||
"scrypted-package-json": "scrypted-package-json" | ||
}, | ||
"scrypted": { | ||
"name": "ONNX Object Detection", | ||
"pluginDependencies": [ | ||
"@scrypted/objectdetector" | ||
], | ||
"runtime": "python", | ||
"type": "API", | ||
"interfaces": [ | ||
"Settings", | ||
"ObjectDetection", | ||
"ObjectDetectionPreview" | ||
] | ||
}, | ||
"devDependencies": { | ||
"@scrypted/sdk": "file:../../sdk" | ||
}, | ||
"version": "0.1.79" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../../openvino/src/common |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../../tensorflow-lite/src/detect |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
from ort import ONNXPlugin | ||
|
||
def create_scrypted_plugin(): | ||
return ONNXPlugin() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
from __future__ import annotations | ||
|
||
import asyncio | ||
from typing import Any, Tuple | ||
|
||
import numpy as np | ||
import onnxruntime | ||
import scrypted_sdk | ||
from PIL import Image | ||
import ast | ||
from scrypted_sdk.other import SettingValue | ||
from scrypted_sdk.types import Setting | ||
import concurrent.futures | ||
|
||
import common.yolo as yolo | ||
from predict import PredictPlugin | ||
|
||
predictExecutor = concurrent.futures.ThreadPoolExecutor(1, "ONNX-Predict") | ||
|
||
availableModels = [ | ||
"Default", | ||
"scrypted_yolov6n_320", | ||
"scrypted_yolov6n", | ||
"scrypted_yolov6s_320", | ||
"scrypted_yolov6s", | ||
"scrypted_yolov9c_320", | ||
"scrypted_yolov9c", | ||
"scrypted_yolov8n_320", | ||
"scrypted_yolov8n", | ||
] | ||
|
||
|
||
def parse_labels(names): | ||
j = ast.literal_eval(names) | ||
ret = {} | ||
for k, v in j.items(): | ||
ret[int(k)] = v | ||
return ret | ||
|
||
class ONNXPlugin( | ||
PredictPlugin, scrypted_sdk.BufferConverter, scrypted_sdk.Settings, scrypted_sdk.DeviceProvider | ||
): | ||
def __init__(self, nativeId: str | None = None): | ||
super().__init__(nativeId=nativeId) | ||
|
||
model = self.storage.getItem("model") or "Default" | ||
if model == "Default" or model not in availableModels: | ||
if model != "Default": | ||
self.storage.setItem("model", "Default") | ||
model = "scrypted_yolov8n_320" | ||
self.yolo = "yolo" in model | ||
self.scrypted_yolo = "scrypted_yolo" in model | ||
self.scrypted_model = "scrypted" in model | ||
|
||
print(f"model {model}") | ||
|
||
onnxmodel = "best" if self.scrypted_model else model | ||
|
||
model_version = "v2" | ||
onnxfile = self.downloadFile( | ||
f"https://raw.githubusercontent.com/koush/onnx-models/main/{model}/{onnxmodel}.onnx", | ||
f"{model_version}/{model}/{onnxmodel}.onnx", | ||
) | ||
|
||
print(onnxfile) | ||
|
||
try: | ||
self.compiled_model = onnxruntime.InferenceSession(onnxfile) | ||
except: | ||
import traceback | ||
|
||
traceback.print_exc() | ||
print("Reverting all settings.") | ||
self.storage.removeItem("model") | ||
self.requestRestart() | ||
|
||
input = self.compiled_model.get_inputs()[0] | ||
self.model_dim = input.shape[2] | ||
self.input_name = input.name | ||
self.labels = parse_labels(self.compiled_model.get_modelmeta().custom_metadata_map['names']) | ||
|
||
async def getSettings(self) -> list[Setting]: | ||
model = self.storage.getItem("model") or "Default" | ||
return [ | ||
{ | ||
"key": "model", | ||
"title": "Model", | ||
"description": "The detection model used to find objects.", | ||
"choices": availableModels, | ||
"value": model, | ||
}, | ||
] | ||
|
||
async def putSetting(self, key: str, value: SettingValue): | ||
self.storage.setItem(key, value) | ||
await self.onDeviceEvent(scrypted_sdk.ScryptedInterface.Settings.value, None) | ||
self.requestRestart() | ||
|
||
# width, height, channels | ||
def get_input_details(self) -> Tuple[int, int, int]: | ||
return [self.model_dim, self.model_dim, 3] | ||
|
||
def get_input_size(self) -> Tuple[int, int]: | ||
return [self.model_dim, self.model_dim] | ||
|
||
async def detect_once(self, input: Image.Image, settings: Any, src_size, cvss): | ||
def predict(input_tensor): | ||
output_tensors = self.compiled_model.run(None, { self.input_name: input_tensor }) | ||
objs = yolo.parse_yolov9(output_tensors[0][0]) | ||
return objs | ||
|
||
im = np.array(input) | ||
im = np.stack([input]) | ||
im = im.transpose((0, 3, 1, 2)) # BHWC to BCHW, (n, 3, h, w) | ||
im = im.astype(np.float32) / 255.0 | ||
im = np.ascontiguousarray(im) # contiguous | ||
input_tensor = im | ||
|
||
try: | ||
objs = await asyncio.get_event_loop().run_in_executor( | ||
predictExecutor, lambda: predict(input_tensor) | ||
) | ||
|
||
except: | ||
import traceback | ||
|
||
traceback.print_exc() | ||
raise | ||
|
||
ret = self.create_detection_result(objs, src_size, cvss) | ||
return ret |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
../../tensorflow-lite/src/predict |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
onnxruntime | ||
|
||
# pillow-simd is available on x64 linux | ||
# pillow-simd confirmed not building with arm64 linux or apple silicon | ||
Pillow>=5.4.1; sys_platform != 'linux' or platform_machine != 'x86_64' | ||
pillow-simd; sys_platform == 'linux' and platform_machine == 'x86_64' |
Oops, something went wrong.