From d8276e7c6b77b380ac0b2e520e3e41d43fb66eca Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 18 Oct 2019 14:04:05 -0400 Subject: [PATCH 01/25] Fix wrong param on jsdoc --- src/ImageClassifier/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImageClassifier/index.js b/src/ImageClassifier/index.js index be6a26fbf..a267d2141 100644 --- a/src/ImageClassifier/index.js +++ b/src/ImageClassifier/index.js @@ -27,7 +27,7 @@ const MODEL_OPTIONS = ['mobilenet', 'darknet', 'darknet-tiny', 'doodlenet']; class ImageClassifier { /** * Create an ImageClassifier. - * @param {modelNameOrUrl} modelNameOrUrl - The name or the URL of the model to use. Current model name options + * @param {string} modelNameOrUrl - The name or the URL of the model to use. Current model name options * are: 'mobilenet', 'darknet', 'darknet-tiny', and 'doodlenet'. * @param {HTMLVideoElement} video - An HTMLVideoElement. * @param {object} options - An object with options. From 027ecc7d38d2905737790881b87f9d1eaa296bfc Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 18 Oct 2019 14:29:33 -0400 Subject: [PATCH 02/25] Add deprecation notice for YOLO --- src/YOLO/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/YOLO/index.js b/src/YOLO/index.js index 9a9646c6c..8d465d17f 100644 --- a/src/YOLO/index.js +++ b/src/YOLO/index.js @@ -35,6 +35,9 @@ const DEFAULTS = { const imageSize = 416; class YOLOBase extends Video { + /** + * @deprecated Please use ObjectDetector class instead + */ /** * @typedef {Object} options * @property {number} filterBoxesThreshold - default 0.01 @@ -57,7 +60,8 @@ class YOLOBase extends Video { this.modelReady = false; this.isPredicting = false; this.ready = callCallback(this.loadModel(), callback); - // this.then = this.ready.then; + + console.warn("WARNING! Function YOLO has been deprecated, please use the new ObjectDetector function instead"); } async loadModel() { From dd816b7275c7694cce1e78e56f945ddb306ca517 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 18 Oct 2019 14:36:09 -0400 Subject: [PATCH 03/25] Create new ObjectDetector class in place of YOLO --- src/ObjectDetector/index.js | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/ObjectDetector/index.js diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js new file mode 100644 index 000000000..be6212968 --- /dev/null +++ b/src/ObjectDetector/index.js @@ -0,0 +1,40 @@ +// Copyright (c) 2019 ml5 +// +// This software is released under the MIT License. +// https://opensource.org/licenses/MIT + +/* + ObjectDetection +*/ + +import YOLO from '../YOLO/index'; + +class ObjectDetectorBase { + /** + * @typedef {Object} options + * @property {number} filterBoxesThreshold - default 0.01 + * @property {number} IOUThreshold - default 0.4 + * @property {number} classProbThreshold - default 0.4 + */ + /** + * Create ObjectDetector model. Works on video and images. + * @param {string} modelName - The name or the URL of the model to use. Current model name options + * are: 'YOLO'. + * @param {HTMLVideoElement} video - The video to be used for object detection and classification. + * @param {Object} options - Optional. A set of options. + * @param {function} callback - Optional. A callback function that is called once the model has loaded. + * If no callback is provided, it will return a promise that will be resolved once the model has loaded. + */ + constructor(modelName, video, options, callback) { + this.modelName = modelName; + this.video = video; + this.options = options; + this.callback = callback; + } +} + +const ObjectDetector = (modelName, video, options, callback) => { + return new YOLO(video, options, callback); +}; + +export default ObjectDetector; From 1a5880fbce8ef40a3c624eab6fba3c834f1233af Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 15:05:56 -0400 Subject: [PATCH 04/25] Add ObjectDetector to main export --- src/ObjectDetector/index.js | 6 +++--- src/index.js | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index be6212968..676893084 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -12,9 +12,9 @@ import YOLO from '../YOLO/index'; class ObjectDetectorBase { /** * @typedef {Object} options - * @property {number} filterBoxesThreshold - default 0.01 - * @property {number} IOUThreshold - default 0.4 - * @property {number} classProbThreshold - default 0.4 + * @property {number} filterBoxesThreshold - Optional. default 0.01 + * @property {number} IOUThreshold - Optional. default 0.4 + * @property {number} classProbThreshold - Optional. default 0.4 */ /** * Create ObjectDetector model. Works on video and images. diff --git a/src/index.js b/src/index.js index 22e8d95b5..815c4a041 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,7 @@ import KNNClassifier from './KNNClassifier/'; import featureExtractor from './FeatureExtractor/'; import word2vec from './Word2vec/'; import YOLO from './YOLO'; +import ObjectDetector from './ObjectDetector'; import poseNet from './PoseNet'; import * as imageUtils from './utils/imageUtilities'; import styleTransfer from './StyleTransfer/'; @@ -43,6 +44,7 @@ const withPreload = { styleTransfer, word2vec, YOLO, + ObjectDetector, uNet, }; From 0c54d019d39b57127ff56fbab4c94ad45c6d8844 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 15:07:59 -0400 Subject: [PATCH 05/25] Move YOLO folder ro ObjectDetection --- src/{ => ObjectDetector}/YOLO/index.js | 8 ++++---- src/{ => ObjectDetector}/YOLO/index_test.js | 0 src/{ => ObjectDetector}/YOLO/postprocess.js | 0 src/ObjectDetector/index.js | 2 +- src/index.js | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename src/{ => ObjectDetector}/YOLO/index.js (96%) rename src/{ => ObjectDetector}/YOLO/index_test.js (100%) rename src/{ => ObjectDetector}/YOLO/postprocess.js (100%) diff --git a/src/YOLO/index.js b/src/ObjectDetector/YOLO/index.js similarity index 96% rename from src/YOLO/index.js rename to src/ObjectDetector/YOLO/index.js index 8d465d17f..5857ccdd6 100644 --- a/src/YOLO/index.js +++ b/src/ObjectDetector/YOLO/index.js @@ -10,10 +10,10 @@ Heavily derived from https://github.com/ModelDepot/tfjs-yolo-tiny (ModelDepot: m */ import * as tf from '@tensorflow/tfjs'; -import Video from '../utils/Video'; -import { imgToTensor } from '../utils/imageUtilities'; -import callCallback from '../utils/callcallback'; -import CLASS_NAMES from './../utils/COCO_CLASSES'; +import Video from '../../utils/Video'; +import { imgToTensor } from '../../utils/imageUtilities'; +import callCallback from '../../utils/callcallback'; +import CLASS_NAMES from '../../utils/COCO_CLASSES'; import { nonMaxSuppression, diff --git a/src/YOLO/index_test.js b/src/ObjectDetector/YOLO/index_test.js similarity index 100% rename from src/YOLO/index_test.js rename to src/ObjectDetector/YOLO/index_test.js diff --git a/src/YOLO/postprocess.js b/src/ObjectDetector/YOLO/postprocess.js similarity index 100% rename from src/YOLO/postprocess.js rename to src/ObjectDetector/YOLO/postprocess.js diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 676893084..bf46dfab5 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -7,7 +7,7 @@ ObjectDetection */ -import YOLO from '../YOLO/index'; +import YOLO from './YOLO/index'; class ObjectDetectorBase { /** diff --git a/src/index.js b/src/index.js index 815c4a041..6ad485456 100644 --- a/src/index.js +++ b/src/index.js @@ -10,7 +10,7 @@ import soundClassifier from './SoundClassifier/'; import KNNClassifier from './KNNClassifier/'; import featureExtractor from './FeatureExtractor/'; import word2vec from './Word2vec/'; -import YOLO from './YOLO'; +import YOLO from './ObjectDetector/YOLO'; import ObjectDetector from './ObjectDetector'; import poseNet from './PoseNet'; import * as imageUtils from './utils/imageUtilities'; From 37457f675b151e7ff99118c6a6e7fbc9fbf7368b Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 15:37:41 -0400 Subject: [PATCH 06/25] Stop displaying deprecation notice when using ObjectDetector --- src/ObjectDetector/YOLO/index.js | 4 +++- src/ObjectDetector/index.js | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ObjectDetector/YOLO/index.js b/src/ObjectDetector/YOLO/index.js index 5857ccdd6..e278b88f3 100644 --- a/src/ObjectDetector/YOLO/index.js +++ b/src/ObjectDetector/YOLO/index.js @@ -61,7 +61,9 @@ class YOLOBase extends Video { this.isPredicting = false; this.ready = callCallback(this.loadModel(), callback); - console.warn("WARNING! Function YOLO has been deprecated, please use the new ObjectDetector function instead"); + if (!options.disableDeprecationNotice) { + console.warn("WARNING! Function YOLO has been deprecated, please use the new ObjectDetector function instead"); + } } async loadModel() { diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index bf46dfab5..d397118a5 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -28,12 +28,13 @@ class ObjectDetectorBase { constructor(modelName, video, options, callback) { this.modelName = modelName; this.video = video; - this.options = options; + this.options = options || {}; this.callback = callback; } } const ObjectDetector = (modelName, video, options, callback) => { + options.disableDeprecationNotice = true; return new YOLO(video, options, callback); }; From fcab242ab9a8ef6ef9bcedf39b247852408e84f6 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 16:04:52 -0400 Subject: [PATCH 07/25] Add tf js coco ssd dependency --- package-lock.json | 5 +++++ package.json | 1 + 2 files changed, 6 insertions(+) diff --git a/package-lock.json b/package-lock.json index 0ff71296b..3ca1b712e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -324,6 +324,11 @@ "resolved": "https://registry.npmjs.org/@tensorflow-models/body-pix/-/body-pix-1.1.2.tgz", "integrity": "sha512-moCCTlP77v20HMg1e/Hs1LehCDLAKS32e6OUeI1MA/4HrRRO1Dq9engVCLFZUMO2+mJXdQeBdzexcFg0WQox7w==" }, + "@tensorflow-models/coco-ssd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tensorflow-models/coco-ssd/-/coco-ssd-2.0.0.tgz", + "integrity": "sha512-JexMswea9a5k8//sdImcxdVaofejkzUzJA8cpDBOZSauBkBM34M3c1YoEUrFuJSa5sKxgMu5TwMPWbJ59A7IVA==" + }, "@tensorflow-models/knn-classifier": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@tensorflow-models/knn-classifier/-/knn-classifier-1.2.1.tgz", diff --git a/package.json b/package.json index 6b8469e36..393b203c9 100644 --- a/package.json +++ b/package.json @@ -102,6 +102,7 @@ "dependencies": { "@magenta/sketch": "0.2.0", "@tensorflow-models/body-pix": "1.1.2", + "@tensorflow-models/coco-ssd": "^2.0.0", "@tensorflow-models/knn-classifier": "1.2.1", "@tensorflow-models/mobilenet": "2.0.3", "@tensorflow-models/posenet": "2.1.3", From 7eab584f6a17604571ca29290897817374205a70 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 17:43:30 -0400 Subject: [PATCH 08/25] WIP adding CocoSsd functionality --- src/ObjectDetector/CocoSsd/index.js | 55 +++++++++++++++++++++++++++++ src/ObjectDetector/index.js | 24 +++++++++---- 2 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 src/ObjectDetector/CocoSsd/index.js diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js new file mode 100644 index 000000000..34fa66f2d --- /dev/null +++ b/src/ObjectDetector/CocoSsd/index.js @@ -0,0 +1,55 @@ +// Copyright (c) 2018 ml5 +// +// This software is released under the MIT License. +// https://opensource.org/licenses/MIT +/* eslint max-len: ["error", { "code": 180 }] */ + +/* + COCO-SSD Object detection + Wraps the coco-ssd model in tfjs to be used in ml5 +*/ + +import * as tf from '@tensorflow/tfjs'; +import * as cocoSsd from '@tensorflow-models/coco-ssd'; + +class CocoSsd { + /** + * Create CocoSsd model. Works on video and images. + * @param {HTMLVideoElement} video - Optional. The video to be used for object detection and classification. + * @param {Object} options - Optional. A set of options. + * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise + * that will be resolved once the model has loaded. + */ + constructor(video, options, callback) { + this.isModelReady = false; + this.video = video; + this.options = options; + this.callback = callback; + cocoSsd.load().then(_cocoSsdModel => { + this.cocoSsdModel = _cocoSsdModel; + callback(); + }); + } + + detect(callback) { + if (this.isModelReady) { + this.cocoSsdModel.detect(this.video).then((predictions) => { + let formattedPredictions = []; + for (let i = 0; i < predictions.length; i++) { + const prediction = predictions[i]; + formattedPredictions.push({ + label: prediction.class, + confidence: prediction.score, + x: prediction.bbox[0], + y: prediction.bbox[1], + w: prediction.bbox[2], + h: prediction.bbox[3], + }); + } + callback(false, formattedPredictions); + }) + } + } +} + +export default CocoSsd; diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index d397118a5..1903fd33f 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -8,8 +8,9 @@ */ import YOLO from './YOLO/index'; +import CocoSsd from './CocoSsd/index'; -class ObjectDetectorBase { +class ObjectDetector { /** * @typedef {Object} options * @property {number} filterBoxesThreshold - Optional. default 0.01 @@ -30,12 +31,23 @@ class ObjectDetectorBase { this.video = video; this.options = options || {}; this.callback = callback; + + switch (modelName) { + case 'YOLO': + options.disableDeprecationNotice = true; + this.model = new YOLO(video, options, callback); + break; + case 'CocoSsd': + this.model = new CocoSsd(video, options, callback); + break; + default: + throw new Error('Model name not supported') + } } -} -const ObjectDetector = (modelName, video, options, callback) => { - options.disableDeprecationNotice = true; - return new YOLO(video, options, callback); -}; + detect(callback) { + this.model.detect(callback); + } +} export default ObjectDetector; From 0d63b37283f3e3f25b074fc129a48479f565dd67 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 18:06:23 -0400 Subject: [PATCH 09/25] fix linter issue --- src/ObjectDetector/CocoSsd/index.js | 5 ++--- src/ObjectDetector/index.js | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index 34fa66f2d..f9ed586e6 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -9,7 +9,6 @@ Wraps the coco-ssd model in tfjs to be used in ml5 */ -import * as tf from '@tensorflow/tfjs'; import * as cocoSsd from '@tensorflow-models/coco-ssd'; class CocoSsd { @@ -34,8 +33,8 @@ class CocoSsd { detect(callback) { if (this.isModelReady) { this.cocoSsdModel.detect(this.video).then((predictions) => { - let formattedPredictions = []; - for (let i = 0; i < predictions.length; i++) { + const formattedPredictions = []; + for (let i = 0; i < predictions.length; i += 1) { const prediction = predictions[i]; formattedPredictions.push({ label: prediction.class, diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 1903fd33f..2972753bf 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -34,8 +34,7 @@ class ObjectDetector { switch (modelName) { case 'YOLO': - options.disableDeprecationNotice = true; - this.model = new YOLO(video, options, callback); + this.model = new YOLO(video, { disableDeprecationNotice: true, ...options }, callback); break; case 'CocoSsd': this.model = new CocoSsd(video, options, callback); From 75303dd87f2db41cb655e0832f1271652ce1ffb9 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 18:41:24 -0400 Subject: [PATCH 10/25] WIP get coco-ssd working with callbacks --- src/ObjectDetector/CocoSsd/index.js | 2 ++ src/ObjectDetector/index.js | 7 ++++++- src/index.js | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index f9ed586e6..36e6b63f6 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -26,11 +26,13 @@ class CocoSsd { this.callback = callback; cocoSsd.load().then(_cocoSsdModel => { this.cocoSsdModel = _cocoSsdModel; + this.isModelReady = true; callback(); }); } detect(callback) { + console.log('detect at CocoSsd') if (this.isModelReady) { this.cocoSsdModel.detect(this.video).then((predictions) => { const formattedPredictions = []; diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 2972753bf..e195ffa35 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -45,8 +45,13 @@ class ObjectDetector { } detect(callback) { + console.log('detect at ObjectDetector') this.model.detect(callback); } } -export default ObjectDetector; +const objectDetector = (modelName, video, options, callback) => { + return new ObjectDetector(modelName, video, options, callback) +} + +export default objectDetector; diff --git a/src/index.js b/src/index.js index 6ad485456..3729e2a10 100644 --- a/src/index.js +++ b/src/index.js @@ -11,7 +11,7 @@ import KNNClassifier from './KNNClassifier/'; import featureExtractor from './FeatureExtractor/'; import word2vec from './Word2vec/'; import YOLO from './ObjectDetector/YOLO'; -import ObjectDetector from './ObjectDetector'; +import objectDetector from './ObjectDetector'; import poseNet from './PoseNet'; import * as imageUtils from './utils/imageUtilities'; import styleTransfer from './StyleTransfer/'; @@ -44,7 +44,7 @@ const withPreload = { styleTransfer, word2vec, YOLO, - ObjectDetector, + objectDetector, uNet, }; From 6d93030832a5cd463873f8f2c212f90f96d21cfa Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 19:18:56 -0400 Subject: [PATCH 11/25] Support p5 video for coco ssd --- src/ObjectDetector/CocoSsd/index.js | 1 - src/ObjectDetector/index.js | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index 36e6b63f6..42564e173 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -32,7 +32,6 @@ class CocoSsd { } detect(callback) { - console.log('detect at CocoSsd') if (this.isModelReady) { this.cocoSsdModel.detect(this.video).then((predictions) => { const formattedPredictions = []; diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index e195ffa35..5ae405c83 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -32,12 +32,18 @@ class ObjectDetector { this.options = options || {}; this.callback = callback; + if (video instanceof HTMLVideoElement) { + this.video = video; + } else if (typeof video === 'object' && video.elt instanceof HTMLVideoElement) { + this.video = video.elt; // Handle p5.js image + } + switch (modelName) { case 'YOLO': - this.model = new YOLO(video, { disableDeprecationNotice: true, ...options }, callback); + this.model = new YOLO(this.video, { disableDeprecationNotice: true, ...options }, callback); break; case 'CocoSsd': - this.model = new CocoSsd(video, options, callback); + this.model = new CocoSsd(this.video, options, callback); break; default: throw new Error('Model name not supported') From cfe5fc7b3aa7aec87941bc97f007efa6e9c7ae2d Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 20:39:18 -0400 Subject: [PATCH 12/25] Add canvas support for YOLO --- src/ObjectDetector/YOLO/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ObjectDetector/YOLO/index.js b/src/ObjectDetector/YOLO/index.js index e278b88f3..833b701cd 100644 --- a/src/ObjectDetector/YOLO/index.js +++ b/src/ObjectDetector/YOLO/index.js @@ -82,6 +82,7 @@ class YOLOBase extends Video { if (inputOrCallback instanceof HTMLImageElement || inputOrCallback instanceof HTMLVideoElement + || inputOrCallback instanceof HTMLCanvasElement || inputOrCallback instanceof ImageData) { imgToPredict = inputOrCallback; } else if (typeof inputOrCallback === 'object' && (inputOrCallback.elt instanceof HTMLImageElement From a8622b23a3a1b46bfe2503061bb13dfcaed38210 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 20:40:24 -0400 Subject: [PATCH 13/25] Add canvas image and video support for ObjectDetector --- src/ObjectDetector/CocoSsd/index.js | 14 +++++++++----- src/ObjectDetector/index.js | 16 +++++++++------- src/utils/imageUtilities.js | 9 +++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index 42564e173..c06b26cc4 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -14,14 +14,12 @@ import * as cocoSsd from '@tensorflow-models/coco-ssd'; class CocoSsd { /** * Create CocoSsd model. Works on video and images. - * @param {HTMLVideoElement} video - Optional. The video to be used for object detection and classification. * @param {Object} options - Optional. A set of options. * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise * that will be resolved once the model has loaded. */ - constructor(video, options, callback) { + constructor(options, callback) { this.isModelReady = false; - this.video = video; this.options = options; this.callback = callback; cocoSsd.load().then(_cocoSsdModel => { @@ -31,9 +29,15 @@ class CocoSsd { }); } - detect(callback) { + /** + * Detect objects that are in video, returns bounding box, label, and confidence scores + * @param {HTMLVideoElement|HTMLImageElement|HTMLCanvasElement|ImageData} subject - Subject of the detection. + * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise + * that will be resolved once the prediction is done. + */ + detect(subject, callback) { if (this.isModelReady) { - this.cocoSsdModel.detect(this.video).then((predictions) => { + this.cocoSsdModel.detect(subject).then((predictions) => { const formattedPredictions = []; for (let i = 0; i < predictions.length; i += 1) { const prediction = predictions[i]; diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 5ae405c83..5da84408a 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -9,6 +9,7 @@ import YOLO from './YOLO/index'; import CocoSsd from './CocoSsd/index'; +import { isInstanceOfSupportedElement } from '../utils/imageUtilities'; class ObjectDetector { /** @@ -32,18 +33,20 @@ class ObjectDetector { this.options = options || {}; this.callback = callback; - if (video instanceof HTMLVideoElement) { + if (isInstanceOfSupportedElement(video)) { + console.log('element assigned A') this.video = video; - } else if (typeof video === 'object' && video.elt instanceof HTMLVideoElement) { - this.video = video.elt; // Handle p5.js image + } else if (typeof video === 'object' && isInstanceOfSupportedElement(video.elt)) { + console.log('element assigned B') + this.video = video.elt; // Handle p5.js video and image } switch (modelName) { case 'YOLO': - this.model = new YOLO(this.video, { disableDeprecationNotice: true, ...options }, callback); + this.model = new YOLO({ disableDeprecationNotice: true, ...options }, callback); break; case 'CocoSsd': - this.model = new CocoSsd(this.video, options, callback); + this.model = new CocoSsd(options, callback); break; default: throw new Error('Model name not supported') @@ -51,8 +54,7 @@ class ObjectDetector { } detect(callback) { - console.log('detect at ObjectDetector') - this.model.detect(callback); + this.model.detect(this.video, callback); } } diff --git a/src/utils/imageUtilities.js b/src/utils/imageUtilities.js index 6122c6189..31ee886a1 100644 --- a/src/utils/imageUtilities.js +++ b/src/utils/imageUtilities.js @@ -74,9 +74,18 @@ function imgToTensor(input, size = null) { }); } +// Static Method: Check if subject is supported for object detection +function isInstanceOfSupportedElement(subject) { + return (subject instanceof HTMLVideoElement + || subject instanceof HTMLImageElement + || subject instanceof HTMLCanvasElement + || subject instanceof ImageData) +} + export { array3DToImage, processVideo, cropImage, imgToTensor, + isInstanceOfSupportedElement, }; From 7a0d43aa068e400831b5a3858b50cefad055adb7 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 21 Oct 2019 21:28:40 -0400 Subject: [PATCH 14/25] Return a promis for objectdetections prediciton --- src/ObjectDetector/CocoSsd/index.js | 4 ++-- src/ObjectDetector/index.js | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index c06b26cc4..cc72c0d87 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -35,7 +35,7 @@ class CocoSsd { * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise * that will be resolved once the prediction is done. */ - detect(subject, callback) { + async detect(subject, callback) { if (this.isModelReady) { this.cocoSsdModel.detect(subject).then((predictions) => { const formattedPredictions = []; @@ -50,7 +50,7 @@ class CocoSsd { h: prediction.bbox[3], }); } - callback(false, formattedPredictions); + return callback(false, formattedPredictions); }) } } diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 5da84408a..5ab9c7a21 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -34,10 +34,8 @@ class ObjectDetector { this.callback = callback; if (isInstanceOfSupportedElement(video)) { - console.log('element assigned A') this.video = video; } else if (typeof video === 'object' && isInstanceOfSupportedElement(video.elt)) { - console.log('element assigned B') this.video = video.elt; // Handle p5.js video and image } @@ -54,7 +52,7 @@ class ObjectDetector { } detect(callback) { - this.model.detect(this.video, callback); + return this.model.detect(this.video, callback); } } From fed6cdcaf6a591985f53c9db33d195c13d587f0c Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 4 Nov 2019 14:30:04 -0500 Subject: [PATCH 15/25] Allow the use of promises --- src/ObjectDetector/CocoSsd/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index cc72c0d87..84c24c9f9 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -10,6 +10,7 @@ */ import * as cocoSsd from '@tensorflow-models/coco-ssd'; +import callCallback from '../../utils/callcallback'; class CocoSsd { /** @@ -37,7 +38,7 @@ class CocoSsd { */ async detect(subject, callback) { if (this.isModelReady) { - this.cocoSsdModel.detect(subject).then((predictions) => { + return this.cocoSsdModel.detect(subject).then((predictions) => { const formattedPredictions = []; for (let i = 0; i < predictions.length; i += 1) { const prediction = predictions[i]; @@ -50,9 +51,12 @@ class CocoSsd { h: prediction.bbox[3], }); } - return callback(false, formattedPredictions); + return callCallback(new Promise((resolve) => { + resolve(formattedPredictions); + }), callback); }) } + return false; } } From 8565c38a7303aebd5bd62fb5caf7d7430b55b921 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 4 Nov 2019 19:44:48 -0500 Subject: [PATCH 16/25] Remove unused method --- src/utils/imageUtilities.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/utils/imageUtilities.js b/src/utils/imageUtilities.js index 2cc6f34aa..5a6ad2df0 100644 --- a/src/utils/imageUtilities.js +++ b/src/utils/imageUtilities.js @@ -133,14 +133,6 @@ function imgToTensor(input, size = null) { }); } -// Static Method: Check if subject is supported for object detection -function isInstanceOfSupportedElement(subject) { - return (subject instanceof HTMLVideoElement - || subject instanceof HTMLImageElement - || subject instanceof HTMLCanvasElement - || subject instanceof ImageData) -} - export { array3DToImage, processVideo, From 4485a1a39395d5848ffb86446a6d7913d0040c36 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 4 Nov 2019 19:46:26 -0500 Subject: [PATCH 17/25] Fix wrong jsdoc param --- src/ImageClassifier/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/ImageClassifier/index.js b/src/ImageClassifier/index.js index d65a2bcda..d9f99c9e8 100644 --- a/src/ImageClassifier/index.js +++ b/src/ImageClassifier/index.js @@ -27,11 +27,7 @@ const MODEL_OPTIONS = ['mobilenet', 'darknet', 'darknet-tiny', 'doodlenet']; class ImageClassifier { /** * Create an ImageClassifier. -<<<<<<< HEAD * @param {string} modelNameOrUrl - The name or the URL of the model to use. Current model name options -======= - * @param {modelNameOrUrl} modelNameOrUrl - The name or the URL of the model to use. Current model name options ->>>>>>> 2d35106a7ebacb0bf9ad2e00374c4558abc6781a * are: 'mobilenet', 'darknet', 'darknet-tiny', and 'doodlenet'. * @param {HTMLVideoElement} video - An HTMLVideoElement. * @param {object} options - An object with options. From 88e0a829a8092c88aed5b23a549a4438dda9e0f9 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Tue, 5 Nov 2019 13:53:24 -0500 Subject: [PATCH 18/25] Refactor and fix promise execution --- src/ObjectDetector/CocoSsd/index.js | 54 ++++++++++++++--------------- src/utils/imageUtilities.js | 8 +++++ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index 84c24c9f9..ecb6c3f39 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -2,7 +2,6 @@ // // This software is released under the MIT License. // https://opensource.org/licenses/MIT -/* eslint max-len: ["error", { "code": 180 }] */ /* COCO-SSD Object detection @@ -16,17 +15,18 @@ class CocoSsd { /** * Create CocoSsd model. Works on video and images. * @param {Object} options - Optional. A set of options. - * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise + * @param {function} constructorCallback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise * that will be resolved once the model has loaded. */ - constructor(options, callback) { - this.isModelReady = false; + constructor(options, constructorCallback) { this.options = options; - this.callback = callback; - cocoSsd.load().then(_cocoSsdModel => { + this.constructorCallback = constructorCallback; + this.ready = callCallback(this.loadModel(), constructorCallback); + } + + async loadModel() { + await cocoSsd.load().then(_cocoSsdModel => { this.cocoSsdModel = _cocoSsdModel; - this.isModelReady = true; - callback(); }); } @@ -37,26 +37,24 @@ class CocoSsd { * that will be resolved once the prediction is done. */ async detect(subject, callback) { - if (this.isModelReady) { - return this.cocoSsdModel.detect(subject).then((predictions) => { - const formattedPredictions = []; - for (let i = 0; i < predictions.length; i += 1) { - const prediction = predictions[i]; - formattedPredictions.push({ - label: prediction.class, - confidence: prediction.score, - x: prediction.bbox[0], - y: prediction.bbox[1], - w: prediction.bbox[2], - h: prediction.bbox[3], - }); - } - return callCallback(new Promise((resolve) => { - resolve(formattedPredictions); - }), callback); - }) - } - return false; + await this.ready; + return this.cocoSsdModel.detect(subject).then((predictions) => { + const formattedPredictions = []; + for (let i = 0; i < predictions.length; i += 1) { + const prediction = predictions[i]; + formattedPredictions.push({ + label: prediction.class, + confidence: prediction.score, + x: prediction.bbox[0], + y: prediction.bbox[1], + w: prediction.bbox[2], + h: prediction.bbox[3], + }); + } + return callCallback(new Promise((resolve) => { + resolve(formattedPredictions); + }), callback); + }) } } diff --git a/src/utils/imageUtilities.js b/src/utils/imageUtilities.js index 5a6ad2df0..4c55caab3 100644 --- a/src/utils/imageUtilities.js +++ b/src/utils/imageUtilities.js @@ -133,10 +133,18 @@ function imgToTensor(input, size = null) { }); } +function isInstanceOfSupportedElement(subject) { + return (subject instanceof HTMLVideoElement + || subject instanceof HTMLImageElement + || subject instanceof HTMLCanvasElement + || subject instanceof ImageData) +} + export { array3DToImage, processVideo, cropImage, imgToTensor, + isInstanceOfSupportedElement, flipImage }; From cb435a696c66026af54ecea8d1e748e24eaa9828 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Tue, 5 Nov 2019 14:54:49 -0500 Subject: [PATCH 19/25] use new format for detection output --- src/ObjectDetector/CocoSsd/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index ecb6c3f39..9529203f5 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -45,10 +45,10 @@ class CocoSsd { formattedPredictions.push({ label: prediction.class, confidence: prediction.score, - x: prediction.bbox[0], - y: prediction.bbox[1], - w: prediction.bbox[2], - h: prediction.bbox[3], + x: prediction.bbox[0] / subject.width, + y: prediction.bbox[1] / subject.height, + w: prediction.bbox[2] / subject.width, + h: prediction.bbox[3] / subject.height, }); } return callCallback(new Promise((resolve) => { From 0a9746e7cccf8be1f906ecc4145ab8be85b8a81b Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Tue, 5 Nov 2019 15:04:43 -0500 Subject: [PATCH 20/25] Add some more jsdoc to make contract easier to understand --- src/ObjectDetector/index.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 5ab9c7a21..28d899bb9 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -51,6 +51,21 @@ class ObjectDetector { } } + /** + * @typedef {Object} ObjectDetectorPrediction + * @property {number} x - top left x coordinate of the prediction box (0 to 1). + * @property {number} y - top left y coordinate of the prediction box (0 to 1). + * @property {number} w - width of the prediction box (0 to 1). + * @property {number} h - height of the prediction box (0 to 1). + * @property {string} label - the label given. + * @property {number} confidence - the confidence score (0 to 1). + */ + /** + * Returns an rgb array + * @param {function} callback - Optional. A callback that deliver the result. If no callback is + * given, a promise is will be returned. + * @return {ObjectDetectorPrediction[]} an array of the prediction result + */ detect(callback) { return this.model.detect(this.video, callback); } From 457d4605f7b2611d7294ea3112b667835c141a36 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Mon, 11 Nov 2019 13:18:13 -0500 Subject: [PATCH 21/25] Remove uneeded passing of option hash --- src/ObjectDetector/CocoSsd/index.js | 6 ++---- src/ObjectDetector/index.js | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/ObjectDetector/CocoSsd/index.js b/src/ObjectDetector/CocoSsd/index.js index 9529203f5..6dac986f7 100644 --- a/src/ObjectDetector/CocoSsd/index.js +++ b/src/ObjectDetector/CocoSsd/index.js @@ -1,4 +1,4 @@ -// Copyright (c) 2018 ml5 +// Copyright (c) 2019 ml5 // // This software is released under the MIT License. // https://opensource.org/licenses/MIT @@ -14,12 +14,10 @@ import callCallback from '../../utils/callcallback'; class CocoSsd { /** * Create CocoSsd model. Works on video and images. - * @param {Object} options - Optional. A set of options. * @param {function} constructorCallback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise * that will be resolved once the model has loaded. */ - constructor(options, constructorCallback) { - this.options = options; + constructor(constructorCallback) { this.constructorCallback = constructorCallback; this.ready = callCallback(this.loadModel(), constructorCallback); } diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index 28d899bb9..d09e9ed85 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -44,7 +44,7 @@ class ObjectDetector { this.model = new YOLO({ disableDeprecationNotice: true, ...options }, callback); break; case 'CocoSsd': - this.model = new CocoSsd(options, callback); + this.model = new CocoSsd(callback); break; default: throw new Error('Model name not supported') From db1b986e05b9720d57e40d8af9121c890fb8cfad Mon Sep 17 00:00:00 2001 From: Joey Lee Date: Thu, 14 Nov 2019 15:17:39 -0500 Subject: [PATCH 22/25] [v0.4.3] v0.4.3 (#702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [devops] Merges fixes in release to development (#698) * [v0.4.2] v0.4.2 (#685) * [neural net fix] Merges neural net fix in release to development branch (#676) * ml5 release v0.4.0 (#640) * Updates dev with release and Removes /dist from past commits and future dev (#479) * Remove unnecessary tf.tidy call * Implement (err, value) callback style, add test for LSTMGenerator * Callbackify Mobilenet.classify and Mobilenet.regressor * Add events dep * add travis.yml * remove circleci * removing test script * removing package-lock.json * update yarn.lock * set up browserstack * snapshot * yarn test-travis from .travis.yml * chmod +x trest-travis * shebang * bs launcher * access key * add a log * kick travis * snapshot * change browserstack env names * Added Browserstack badge * Resolve https://github.com/ml5js/ml5-library/pull/151#discussion_r196475143 * Address https://github.com/ml5js/ml5-library/pull/151#discussion_r196484426 * Address https://github.com/ml5js/ml5-library/pull/151#discussion_r196517984 * Address https://github.com/ml5js/ml5-library/pull/151#discussion_r196518890 * Add license to callCallback, return this from ImageClassifier#loadModel * Add mocha reporter back, try to make classes thenable * Update karma configuration - Fixes a weird bug where tensors were spuriously disposed by tfjs in testing environment due to there being multiple tfjs instances present on the page from the different builds - Improves stack traces to make debugging easier by disabling minification in karma webpack. - Only webpack the main package, this means that the test files have to use the global ml5 object. * Add tests for Word2Vec - validates that there are no leaked tensors - validates basic functionality of nearest - validates that add, subtract and average return things - Fixes memory leaks in add, subtract, average, and addOrSubtract functions - Adds a general dispose to the Word2Vec class * Remove console.log * Add an easy way to run test suite once * Handle rejected promises * fix eslint errors in lstm and w2v * Support for callbacks in word2vec and fix tests. Moved the YOLO weights to the examples repo. Fix eslint in some files * Add tests to imageclassifier fix video issue in imageclassifier * - Add test for YOLO - Add test for Imageclassifier - Resize img if necessary in utils * - Add license to test files - Style transfer test - remove .test file in posenet * - Update poseNet version to 0.1.3 - Add posnet test * test commit * - Fix issues in imageclassifier * - update pitch detection to #158 - fix event emmiter in posenet - fix constructor in yolo - async video util * temp fix to test leak * Fix posenet on video ready * refactor pix2pix to support promises * support promise when create a new styletransfer method * bump to version 0.1.0 * bump to 0.1.1 * update readme * clarification and formatting changes to ensure steps are easy to follow * ImageClassifier: predict: handle unsupported input * add proper attributions for models * Update index.js * fix eslint for links * Update index.js * Update index.js * Update index.js * Update README.md (#207) * Support canvas elements and canvases from p5.js in ImageClassifier (#206) * Bugfix: PitchDetection returning the wrong pitch (#187) When converted to the current syntax, the operation was the wrong way around. See https://github.com/marl/crepe/blob/gh-pages/crepe.js#L141 for the original. * fix(addImage, classify, predict): now featureExtractor works with images (it was working only with video) (#191) * fix(addImage, classify, predict): now featureExtractor works with images (it was working only with v * fix(addImage, classify, predict): now featureExtractor works with images (it was working only with video * added darknet reference and darknet tiny classifiers (#201) * added darknet reference and darknet tiny classifiers * fixed linting stuff * edits.. * edits2.0 * clean comments * SketchRNN (#189) * init sketchrnn * strokes * add models * stylefixes * some more bugfixes * comments * update dependencies * Load and save a custom model created with FeatureExtractor (#219) * load and save feature extractor model * update learning rate * add string an callback * KNN Classifier (#171) * added KNNClassifier class, and getFeatures method in ImageClassifier * added knnclassifer.loaddataset, and savedataset from/as json file * added featureExtractor.infer() to take in p5 videoelement * support callback, and add 'label' for each class * added getClassExampleCountByLabel, support label when save/load dataset * update function naming, predictClass to classify, classLabel to label * updates to way of handling strokes (#228) * Fixing up SketchRNN (#232) * Code of Conduct (#225) Just noticing we don't have a code of conduct so adding to the main library repo (we can discuss whether we should put this across all repos -- probably, yes?) This is the [Contributor Covenant](https://www.contributor-covenant.org/). * saving feature extrctor model with ml5Specs (#233) * Stateful LSTM (#234) * allow passing in of lstm state c and h. return more lstm info (c, h, probabilities) * fix LSTM Stateful ness and other tweaks * LSTM probabilities class property * remove weird ind<100 check * remove unnessecary initCells * add LSTM methods state getter, setter and reset * rename to charRNN and update stateful to match discussed api * add callback to feed * fix bug in predict * adding new probabilites object * new build * Additional arguments option for SketchRNN (#237) * Fixing up SketchRNN * arguments fix for SketchRNN interactive * adding build * update version * KNN update (#256) * let knn.addExample() and classify() accept a plain array as input * simplify and shorten knn api function names * rename a variable in clearlabel() * knn.save checks if file name has .json * remove timestamp in the filename for .save() * update package-loack to fix travis test on node 10 * update yarn lock * fix - adds callback support for single functions in posenet re: #244 (#254) * fix(adds callback support for single functions in posenet https://github.com/ml5js/ml5-library/issue adds callback support for single functions in posenet https://github.com/ml5js/ml5-library/issues/244 https://github.com/ml5js/ml5-library/issues/244 * rm package-lock.json * fix(PoseNet/index.js): Add support for PoseNet `.singlePose()` & `.multiPose()` callback #244 Add support for PoseNet `.singlePose()` & `.multiPose()` callback #244 & updated callback error handling #244 * rm error check when returning callback in posenet singlePose & multiPose * fixed indentation * temp rm cb function * added cb function back in * added yarn and package lock * added lock files from ml5 master * updating to version 0.2.0 (#264) * V0.2.1 Tagged Release (#267) * updating to version 0.2.0 * run build process and built lib * Release v0.2.1 - npm publish issue * added github issue template (#275) * add different ways of including ml5js on the HTML header (#268) (We will follow up and update the version to the latest which is v0.2.1) * updated readme specifying versions for lib (#282) * fix: Use Blobs to save files of larger sizes (#271) * fix: Use Blobs to save files of larger sizes fix #270 * use saveBlob() to save knn models, delete io.saveFile(), because saveBlob does the same thing and can handle big files * Show the version number of the library. (#283) Adds the library version from the package.json for each build. * fix issue that server fails to listen dev folder (#287) * feat(KNNClassifier): Add loadData() that enables user to load dat… (#279) * feat(KNNClassifier): Add loadData() that enables user to load dataset from raw data instead of file With load(), file path needs to be provided to load dataset. This means that JSON file that has dataset needs to be located on the server. You can give a way to download dataset using save(), but you cannot give a way to upload that file and load the dataset. This commit adds loadData() that enables user to load dataset from raw data. By combining this method and tag, you can give a way for user to upload dataset file. * Merge load(path, callback) and loadData(data, callback) into load(pathOrJson, callback). * Merge load() and loadData() in a cleaner way without using Symbol. Ref. https://github.com/ml5js/ml5-library/pull/279#issuecomment-465890231 * feat(src/PoseNet/index.js): Ability to access parts of pose by name. Now can simply do pose.nose.x or pose.nose.confidence. Using original PoseNet naming scheme (i.e. rightShoulder). re #245 * Update ImageClassifier and FeatureExtractor to return [{label, confidence}] (#292) * change imageClassifier.predict() to classify(), keep predict until later version * update imageclassifier.classify results to [{label: .., confidence: ..}] * update featureExtractor regressor return {value: 0-1} * update featureextractor.classify to return [{label, confidence}] * Support p5 preload with ml5 functions (#288) * support p5 preload with ml5 functions * make withPreload object clean * an automatic script to initialize experiments dir (#295) * an alternative way to test with experiments dir * revert change for webpack.dev * Change experiments to manual-test * rename folder to manual-test * update .gitignore * patch style transfer for non-square and canvas-derived inputs (#311) * fix preload bugs (#305) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Hi! I noticed that the changes made here: https://github.com/ml5js/ml5-library/pull/295 have not made their way into the documentation! (#318) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Updated CONTRIBUTING.md with npm run manual-test * Hi! I noticed that the changes made here: https://github.com/ml5js/ml5-library/pull/295 have not made their way into the documentation! (#318) (#319) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Updated CONTRIBUTING.md with npm run manual-test * Add tests to CharRNN (#307) (#320) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * Updated CONTRIBUTING docs and README (#322) * updated readme specifying versions for lib * added documentation to contributing.md * added updates * added label and confidence as output for yolo results (#316) * added label and confidence as output * updates yolo test to use label instead of className * Increase test timeout (#321) * added label and confidence as output * updates yolo test to use label instead of className * set jasmine.DEFAULT_TIMEOUT_INTERVAL to 60000ms * Merges in https://github.com/ml5js/ml5-library/pull/313 via master using deprecating-master branch (#323) * Add tests to CharRNN (#307) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * check preload support for other nets and classifiers (#313) Adds specified nets to support preload // TODO: add examples showing appropriate use of preload * added v0.2.2 to package.json in anticipation of new release * Fixing an issue with prettier and eslint (#324) * Fixing an issue with prettier and eslint There is a conflict between arrow syntax rules in airbnb eslint configuration and the way default prettier works in VSCode. These additional setting should help anyone using VSCode with autoformat + prettier to not run into problems building the library. I am open to better ways of handling this of course! (Note the changes in ImageClassifier are trivial and just testing). * adding eslint-config-prettier as dev dependency * trivial change to run tests again * Adjust charrnn test option length (#326) * changed 500 to 100 to shorted test time * fixed .tobe() to match option * changed vocab size to 64 * changed timeout to 12000 * Adjust charrnn length - comment 2nd test for now (#329) * changed 500 to 100 to shorted test time * fixed .tobe() to match option * changed vocab size to 64 * changed timeout to 12000 * comment out second charRNN test for now to allow merging PRs * Forcing magenta dependency to 0.1.2 (#328) The SketchRNN example doesn't work with more recent magenta (0.1.4) probably due to us lagging behind in versions of tf.js? * SketchRNN creation function should be lowercase (#325) This is related to the discussion in #297. * added v0.2.2 to package.json in anticipation of new release (#330) hooray adding release v0.2.2 * Release v0.2.2 πŸŽ‰ (#331) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Hi! I noticed that the changes made here: https://github.com/ml5js/ml5-library/pull/295 have not made their way into the documentation! (#318) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Updated CONTRIBUTING.md with npm run manual-test * Hi! I noticed that the changes made here: https://github.com/ml5js/ml5-library/pull/295 have not made their way into the documentation! (#318) (#319) * Updated issues template and added PR template (#315) * added github issue template * updated issue template * updated ISSUE_TEMPLATE with friendlier wording and structure * added pr template * added branch notes * Updated CONTRIBUTING.md with npm run manual-test * Add tests to CharRNN (#307) (#320) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * Updated CONTRIBUTING docs and README (#322) * updated readme specifying versions for lib * added documentation to contributing.md * added updates * added label and confidence as output for yolo results (#316) * added label and confidence as output * updates yolo test to use label instead of className * Increase test timeout (#321) * added label and confidence as output * updates yolo test to use label instead of className * set jasmine.DEFAULT_TIMEOUT_INTERVAL to 60000ms * Merges in https://github.com/ml5js/ml5-library/pull/313 via master using deprecating-master branch (#323) * Add tests to CharRNN (#307) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * check preload support for other nets and classifiers (#313) Adds specified nets to support preload // TODO: add examples showing appropriate use of preload * added v0.2.2 to package.json in anticipation of new release * Fixing an issue with prettier and eslint (#324) * Fixing an issue with prettier and eslint There is a conflict between arrow syntax rules in airbnb eslint configuration and the way default prettier works in VSCode. These additional setting should help anyone using VSCode with autoformat + prettier to not run into problems building the library. I am open to better ways of handling this of course! (Note the changes in ImageClassifier are trivial and just testing). * adding eslint-config-prettier as dev dependency * trivial change to run tests again * Adjust charrnn test option length (#326) * changed 500 to 100 to shorted test time * fixed .tobe() to match option * changed vocab size to 64 * changed timeout to 12000 * Adjust charrnn length - comment 2nd test for now (#329) * changed 500 to 100 to shorted test time * fixed .tobe() to match option * changed vocab size to 64 * changed timeout to 12000 * comment out second charRNN test for now to allow merging PRs * Forcing magenta dependency to 0.1.2 (#328) The SketchRNN example doesn't work with more recent magenta (0.1.4) probably due to us lagging behind in versions of tf.js? * SketchRNN creation function should be lowercase (#325) This is related to the discussion in #297. * added v0.2.2 to package.json in anticipation of new release (#330) hooray adding release v0.2.2 * added latest v0.2.2 * Some minor README adjustments (#334) * I am de-emphasizing `@latest` as in my experience this can lead to confusion. * I don't think my name needs to be referenced! * V0.2.3 (#341) * ran npm install, added new package json version, and built dist * updated package lock * updated package lock and ran build * add ImageData as valid image type * add CVAE * add latent dim * add random generate * fix p5Image support * fix CVAE parameter * Added a parameter to the save function so that it is possible to add a custom filename to the model that is saved. * Unet fix (#357) Add uNet model and additional fixes - adds uNet model from @zaidalyafeai ✨ - adds preload() for uNet - uses loadImage on window.loadImage vs. window.p5.loadImage * Added sentiment analysis (#339) * Added sentiment analysis * delete files * fixed issues for pull request * add p5 utils (#358) * fix charRNN tests (#349) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * Add tests to CharRNN (#307) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * check preload support for other nets and classifiers (#313) Adds specified nets to support preload // TODO: add examples showing appropriate use of preload * change CharRNN specs to meet time limit, add initial code for videoClassifier * videoClassifier functioning * charRNN functional * fix out of date file * add preload support for cvae (#360) * Update TensorFlow.js to 1.0.2 (#336) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * Adds fixes to PR #332 for tfjs 1.0.2 updates (#366) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * updated package lock * added @tensorflow/tfjs-core as dependency * add graphmodel for infer (#365) * Add DCGAN Model into ml5 (#351) * Create index.js * updated index.js and DCGAN/index.js * DCGAN updates and fixes (#362) * Create index.js * fixed DCGAN errors * updates p5Utils destructuring, fixes linting issues, and updates tfjs to 1.0.2 to match dcgan reqs * fixed cvae * use this.model instead of using model as param to this.compute() * Makes UNET compatible with tfjs 1.0.2 (#367) * added package-lock * updated UNET for use with tfjs 1.0.2 * Makes Sentiment compatible with tfjs 1.0.2 (#368) * added package-lock * rm sentiment-node * changed loadModel to loadLayersModel * Makes CVAE compatible with tfjs 1.0.2 (#369) * added package-lock * updates cvae to tfjs 1.0.2 api * update tfjs to 1.1.2 (#373) * featureExtractor: accept HTML canvas or p5 canvas when addImage(), classify() or predict() * fix: KNNClassifier accepts a number as class index when addExample(features, number) * added check for moz browser ref:https://stackoverflow.com/questions/48623376/typeerror-capturestream-is-not-a-function (#375) This addresses the video capture breaking in YOLO and potentially other video based functions that require the use of .captureStream(). As the .captureStream() function is still experimental, this adds the moz prefix and a browser check to see if we are using firefox or not. * rm todo * updated package-lock.json * Adds label number option to featureExtractor.classification() (#376) * changed numClasses to numLabels * added num label as option to classification() * updated FeatureExtractor Test with numLabels * adds object as param to .classificaiton() * moved options into this.config * fix feature extractor test - add .config * added pose:poseWithParts into .singlePose() (#381) * Adds jsdoc inline-documentation - work in progress (#378) * added jsdoc documentation for imageClassifier * adds dcgan documentation - needs checking * Add jsdoc (#382) * Add jsdocs for CharRNN * Add jsdocs for CVAE * Add jsdocs for FeatureExtractor * Add jsdocs for KNN * Add jsdocs for PitchDetection * Add jsdocs for Pix2pix * Add jsdocs for posenet * Add jsdocs for Sentiment * Add jsdocs for styletransfer * add linebreaks to long lines * added basic docs to sketchRnn * added basic docs to unet * added basic docs to word2vec * added basic yolo docs * Adds V0.3.0 to package.json and Readme for new release (#385) * changed package.json to v0.3.0 * added latest version reference in readme * added lib min - will remove after this release * 🌈 New Release: ml5.js v0.3.0 πŸŽ‰ (#386) * add ImageData as valid image type * add CVAE * add latent dim * add random generate * fix p5Image support * fix CVAE parameter * Added a parameter to the save function so that it is possible to add a custom filename to the model that is saved. * Unet fix (#357) Add uNet model and additional fixes - adds uNet model from @zaidalyafeai ✨ - adds preload() for uNet - uses loadImage on window.loadImage vs. window.p5.loadImage * Added sentiment analysis (#339) * Added sentiment analysis * delete files * fixed issues for pull request * add p5 utils (#358) * fix charRNN tests (#349) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * Add tests to CharRNN (#307) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * check preload support for other nets and classifiers (#313) Adds specified nets to support preload // TODO: add examples showing appropriate use of preload * change CharRNN specs to meet time limit, add initial code for videoClassifier * videoClassifier functioning * charRNN functional * fix out of date file * add preload support for cvae (#360) * Update TensorFlow.js to 1.0.2 (#336) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * Adds fixes to PR #332 for tfjs 1.0.2 updates (#366) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * updated package lock * added @tensorflow/tfjs-core as dependency * add graphmodel for infer (#365) * Add DCGAN Model into ml5 (#351) * Create index.js * updated index.js and DCGAN/index.js * DCGAN updates and fixes (#362) * Create index.js * fixed DCGAN errors * updates p5Utils destructuring, fixes linting issues, and updates tfjs to 1.0.2 to match dcgan reqs * fixed cvae * use this.model instead of using model as param to this.compute() * Makes UNET compatible with tfjs 1.0.2 (#367) * added package-lock * updated UNET for use with tfjs 1.0.2 * Makes Sentiment compatible with tfjs 1.0.2 (#368) * added package-lock * rm sentiment-node * changed loadModel to loadLayersModel * Makes CVAE compatible with tfjs 1.0.2 (#369) * added package-lock * updates cvae to tfjs 1.0.2 api * update tfjs to 1.1.2 (#373) * featureExtractor: accept HTML canvas or p5 canvas when addImage(), classify() or predict() * fix: KNNClassifier accepts a number as class index when addExample(features, number) * added check for moz browser ref:https://stackoverflow.com/questions/48623376/typeerror-capturestream-is-not-a-function (#375) This addresses the video capture breaking in YOLO and potentially other video based functions that require the use of .captureStream(). As the .captureStream() function is still experimental, this adds the moz prefix and a browser check to see if we are using firefox or not. * rm todo * updated package-lock.json * Adds label number option to featureExtractor.classification() (#376) * changed numClasses to numLabels * added num label as option to classification() * updated FeatureExtractor Test with numLabels * adds object as param to .classificaiton() * moved options into this.config * fix feature extractor test - add .config * added pose:poseWithParts into .singlePose() (#381) * Adds jsdoc inline-documentation - work in progress (#378) * added jsdoc documentation for imageClassifier * adds dcgan documentation - needs checking * Add jsdoc (#382) * Add jsdocs for CharRNN * Add jsdocs for CVAE * Add jsdocs for FeatureExtractor * Add jsdocs for KNN * Add jsdocs for PitchDetection * Add jsdocs for Pix2pix * Add jsdocs for posenet * Add jsdocs for Sentiment * Add jsdocs for styletransfer * add linebreaks to long lines * added basic docs to sketchRnn * added basic docs to unet * added basic docs to word2vec * added basic yolo docs * Adds V0.3.0 to package.json and Readme for new release (#385) * changed package.json to v0.3.0 * added latest version reference in readme * added lib min - will remove after this release * 🌈 New Release: ml5.js v0.3.0 πŸŽ‰ (#386) * add ImageData as valid image type * add CVAE * add latent dim * add random generate * fix p5Image support * fix CVAE parameter * Added a parameter to the save function so that it is possible to add a custom filename to the model that is saved. * Unet fix (#357) Add uNet model and additional fixes - adds uNet model from @zaidalyafeai ✨ - adds preload() for uNet - uses loadImage on window.loadImage vs. window.p5.loadImage * Added sentiment analysis (#339) * Added sentiment analysis * delete files * fixed issues for pull request * add p5 utils (#358) * fix charRNN tests (#349) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * Add tests to CharRNN (#307) * add tests to CharRNN * test(CharRNN): add tests to CharRNN added descriptive tests to ensure CharRNN behaves like its example * remove dist * check preload support for other nets and classifiers (#313) Adds specified nets to support preload // TODO: add examples showing appropriate use of preload * change CharRNN specs to meet time limit, add initial code for videoClassifier * videoClassifier functioning * charRNN functional * fix out of date file * add preload support for cvae (#360) * Update TensorFlow.js to 1.0.2 (#336) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * Adds fixes to PR #332 for tfjs 1.0.2 updates (#366) * upgrade to tfjs1.0.0 * fix loadModel * fix buffer * fix getLayer * updated package lock * added @tensorflow/tfjs-core as dependency * add graphmodel for infer (#365) * Add DCGAN Model into ml5 (#351) * Create index.js * updated index.js and DCGAN/index.js * DCGAN updates and fixes (#362) * Create index.js * fixed DCGAN errors * updates p5Utils destructuring, fixes linting issues, and updates tfjs to 1.0.2 to match dcgan reqs * fixed cvae * use this.model instead of using model as param to this.compute() * Makes UNET compatible with tfjs 1.0.2 (#367) * added package-lock * updated UNET for use with tfjs 1.0.2 * Makes Sentiment compatible with tfjs 1.0.2 (#368) * added package-lock * rm sentiment-node * changed loadModel to loadLayersModel * Makes CVAE compatible with tfjs 1.0.2 (#369) * added package-lock * updates cvae to tfjs 1.0.2 api * update tfjs to 1.1.2 (#373) * featureExtractor: accept HTML canvas or p5 canvas when addImage(), classify() or predict() * fix: KNNClassifier accepts a number as class index when addExample(features, number) * added check for moz browser ref:https://stackoverflow.com/questions/48623376/typeerror-capturestream-is-not-a-function (#375) This addresses the video capture breaking in YOLO and potentially other video based functions that require the use of .captureStream(). As the .captureStream() function is still experimental, this adds the moz prefix and a browser check to see if we are using firefox or not. * rm todo * updated package-lock.json * Adds label number option to featureExtractor.classification() (#376) * changed numClasses to numLabels * added num label as option to classification() * updated FeatureExtractor Test with numLabels * adds object as param to .classificaiton() * moved options into this.config * fix feature extractor test - add .config * added pose:poseWithParts into .singlePose() (#381) * Adds jsdoc inline-documentation - work in progress (#378) * added jsdoc documentation for imageClassifier * adds dcgan documentation - needs checking * Add jsdoc (#382) * Add jsdocs for CharRNN * Add jsdocs for CVAE * Add jsdocs for FeatureExtractor * Add jsdocs for KNN * Add jsdocs for PitchDetection * Add jsdocs for Pix2pix * Add jsdocs for posenet * Add jsdocs for Sentiment * Add jsdocs for styletransfer * add linebreaks to long lines * added basic docs to sketchRnn * added basic docs to unet * added basic docs to word2vec * added basic yolo docs * Adds V0.3.0 to package.json and Readme for new release (#385) * changed package.json to v0.3.0 * added latest version reference in readme * added lib min - will remove after this release * added ml5.imageClassifier(doodleNet) * doodlenet: transfer input img to black white image * warm up model, comments * added ml5.soundClassifier(speech-command) * Revert "added ml5.soundClassifier(speech-command)" This reverts commit 317c146450d65aa1ad9f1ef67bf912a792b0d8a0. * update darknet link to use jsdelivr instead (#391) * changed cdn link to doodlenet model (#390) * adding ml5.soundClassifier(speech-commands) (#389) * added ml5.soundClassifier(speech-commands) * not require fs node module for testing * Reuse getTopKClasses in doodlenet, darkneet and speech-commands * fix: getTopKClasses return className and probability * DCGAN refactor + preload (#398) * restructure dcgan to accept path to manifest.json * added check for absolute or relative url path * rm console.log * added DEFAULT if modelPath is null or undefined in jsonLoader * use async/await in loadModel * add dcgan to preload * fixed breaking change * updated index * added modelReady prop * shows preload working without additional async call to load manifest * allows for preload * added a bird image for testing purpose * update bird image link from ml5-library for testing (#403) * renaming model names to ml5.soundClassifier(SpeechCommands18w) (#404) * ml5.featureExtractor loading models from the new teachable machine (#395) * load model from new teachable machine * featureExtractor use jointModel * only train custom model but save the joint model * Bodypix (#400) * added BodyPix placeholder * "@tensorflow-models/body-pix": "1.0.1", * initial commit placeholder for bodypix * added bodypix as bp and loadModel() * added segment and segmentInternal * change promise to async * return result maskPerson and maskBackground * adds part segmentation * rm log * add segmentation as result.raw * add function to create bodyParts spec object * added test * added optional segmentationOptions param to segment() and segmentWithParts() * added package.lock * updated DEFAULTS to BODYPIX_DEFAULTS * code cleanup * added comments * added jsdoc comments * add check p5 color() * updated bodyPartSpec image color handling for p5 * changed palette spec to object * ml5.imageClassifier() loads pre-trained custom model (#408) * imageClassifier load a model from url * added options to local model from local files * supports imageClassifier.classify(video) * ml5.soundClassifier(url) * docs: add shiffman as a contributor (#410) * docs: update README.md * docs: create .all-contributorsrc * docs: add cvalenzuela as a contributor (#414) * docs: update README.md * docs: create .all-contributorsrc * docs: update README.md * docs: update .all-contributorsrc * rm files to start fresh (#417) * docs: add shiffman as a contributor (#418) * docs: update README.md * docs: create .all-contributorsrc * docs: add cvalenzuela as a contributor (#419) * docs: update README.md * docs: update .all-contributorsrc * docs: add yining1023 as a contributor (#421) * docs: update README.md * docs: update .all-contributorsrc * docs: add handav as a contributor (#423) * docs: update README.md * docs: update .all-contributorsrc * docs: add joeyklee as a contributor (#424) * docs: update README.md * docs: update .all-contributorsrc * docs: add AshleyJaneLewis as a contributor (#425) * docs: update README.md * docs: update .all-contributorsrc * docs: add ellennickles as a contributor (#426) * docs: update README.md * docs: update .all-contributorsrc * docs: add itayniv as a contributor (#427) * docs: update README.md * docs: update .all-contributorsrc * docs: add nikitahuggins as a contributor (#428) * docs: update README.md * docs: update .all-contributorsrc * docs: add AbolTaabol as a contributor (#429) * docs: update README.md * docs: update .all-contributorsrc * docs: add AidanNelson as a contributor (#430) * docs: update README.md * docs: update .all-contributorsrc * docs: add WenheLI as a contributor (#431) * docs: update README.md * docs: update .all-contributorsrc * docs: add dariusk as a contributor (#432) * docs: update README.md * docs: update .all-contributorsrc * docs: add Derek-Wds as a contributor (#433) * docs: update README.md * docs: update .all-contributorsrc * docs: add garym140 as a contributor (#434) * docs: update README.md * docs: update .all-contributorsrc * docs: add genekogan as a contributor (#435) * docs: update README.md * docs: update .all-contributorsrc * docs: add hhayley as a contributor (#436) * docs: update README.md * docs: update .all-contributorsrc * docs: add lisajamhoury as a contributor (#437) * docs: update README.md * docs: update .all-contributorsrc * docs: add matamalaortiz as a contributor (#438) * docs: update README.md * docs: update .all-contributorsrc * docs: add mayaman as a contributor (#439) * docs: update README.md * docs: update .all-contributorsrc * docs: add MimiOnuoha as a contributor (#440) * docs: update README.md * docs: update .all-contributorsrc * docs: add NHibiki as a contributor (#441) * docs: update README.md * docs: update .all-contributorsrc * docs: add oveddan as a contributor (#442) * docs: update README.md * docs: update .all-contributorsrc * docs: add stephkoltun as a contributor (#443) * docs: update README.md * docs: update .all-contributorsrc * docs: add viztopia as a contributor (#444) * docs: update README.md * docs: update .all-contributorsrc * docs: add wenqili as a contributor (#446) * docs: update README.md * docs: update .all-contributorsrc * docs: add brondle as a contributor (#447) * docs: update README.md * docs: update .all-contributorsrc * docs: add Jonarod as a contributor (#448) * docs: update README.md * docs: update .all-contributorsrc * docs: add JazzTap as a contributor (#449) * docs: update README.md * docs: update .all-contributorsrc * docs: add zaidalyafeai as a contributor (#450) * docs: update README.md * docs: update .all-contributorsrc * docs: add AlcaDesign as a contributor (#451) * docs: update README.md * docs: update .all-contributorsrc * docs: add memo as a contributor (#452) * docs: update README.md * docs: update .all-contributorsrc * docs: add TheHidden1 as a contributor (#453) * docs: update README.md * docs: update .all-contributorsrc * docs: add meiamsome as a contributor (#454) * docs: update README.md * docs: update .all-contributorsrc * docs: add marshalhayes as a contributor (#455) * docs: update README.md * docs: update .all-contributorsrc * docs: add reiinakano as a contributor (#456) * docs: update README.md * docs: update .all-contributorsrc * docs: add nsthorat as a contributor (#457) * docs: update README.md * docs: update .all-contributorsrc * docs: add irealva as a contributor (#458) * docs: update README.md * docs: update .all-contributorsrc * docs: add vndrewlee as a contributor (#459) * docs: update README.md * docs: update .all-contributorsrc * docs: add fjcamillo as a contributor (#460) * docs: update README.md * docs: update .all-contributorsrc * docs: add achimkoh as a contributor (#461) * docs: update README.md * docs: update .all-contributorsrc * docs: add hx2A as a contributor (#462) * docs: update README.md * docs: update .all-contributorsrc * docs: add champierre as a contributor (#465) * docs: update README.md * docs: update .all-contributorsrc * docs: add micuat as a contributor (#466) * docs: update README.md * docs: update .all-contributorsrc * docs: add montoyamoraga as a contributor (#467) * docs: update README.md * docs: update .all-contributorsrc * docs: add b2renger as a contributor (#468) * docs: update README.md * docs: update .all-contributorsrc * docs: add adityaas26 as a contributor (#469) * docs: update README.md * docs: update .all-contributorsrc * docs: add okuna291 as a contributor (#470) * docs: update README.md * docs: update .all-contributorsrc * docs: add xujenna as a contributor (#471) * docs: update README.md * docs: update .all-contributorsrc * docs: add nicoleflloyd as a contributor (#472) * docs: update README.md * docs: update .all-contributorsrc * updated package-lock * pass vector to dcgan compute * Adds face-api.js to ml5.js (#482) * initial commit - add classifyExpressions * added classifyExpressionsMultiple() and classifyExpressionsSingle() * remove modelPath requirement in favor of defaults * updated index with links to tiny landmarks * adds option to add relative path or absolute url * updated url checking for models * changed function names * updated functions to .detect() and .detectSingle() * check undefined, otherwise assign * added resize function to results by default * code cleanup * added test to faceapi * added tests * added face parts to result.parts * code cleanup * changed options from withFaceX to withX * code cleanup * code cleanup * fixed conditional to include face .withFaceDescriptor(s) * docs: add jepster-dk as a contributor (#489) * docs: update README.md * docs: update .all-contributorsrc * docs: add xanderjakeq as a contributor (#490) * docs: update README.md * docs: update .all-contributorsrc * docs: add catarak as a contributor (#491) * docs: update README.md * docs: update .all-contributorsrc * [WIP]Change p5Utils into a class * dcgan: use random vector if no latent vector is given * dcgan check if arg is array * updated code of conduct with adapted p5 community statement (#492) * updated code of conduct with adapted p5 community statement * Update CODE_OF_CONDUCT.md rm coding train from donation // todo - add other relevant orgs next Co-Authored-By: Daniel Shiffman * updated enforcement with ashley and dan * rm redundant all contributors badge (#497) * Readme updates (#505) * added package.lock * updates links to latest website * rm 'with no external deps' * set correct url to image (#507) * docs: add dcbriccetti as a contributor (#506) * docs: update README.md * docs: update .all-contributorsrc * docs: add Sblob1 as a contributor (#513) * docs: update README.md * docs: update .all-contributorsrc * docs: update README.md * docs: update .all-contributorsrc * Fu * add command line arguments for testing single model * Handling p5 Image in PoseNet singlePose I have changed singlePose function in order to handle p5 Image types. However I'm not sure if the check is thorough enough. Any pointers are welcome. * Adds ImageData support for all image handling functions (#525) * added ImageData support to YOLO * adds ImageData support to UNET * adds ImageData support to StyleTransfer * adds ImageData support to PoseNet * adds ImageData support for ImageClassifier/darknet * adds ImageData support for ImageClassifier/doodlenet * adds ImageData support for BodyPix and cleans up conditionals * adds ImageData support to faceApi and cleans up conditionals * adds ImageData support for imageclassifier and cleans up conditional * fix image handling for bodypix faceapi and imageclassifier * update mobilenet imageData support * check imagedata in yolo * adds ImageData test to BodyPix * adds check for canvas (#526) * Fixes Posenet input arguments and related issues (#529) * adds this.multiPose() to else{} * fixes conditional check on object or string * added kmeans * added randomSample function * re-ordered name placement * fixed weird indentation error * indentation * removed optional keyword in docstring * updates issue and pull request templates (#530) * Created config class attribute to store instance params * Adds release script helper (#534) * adds release helper function * update name and sep step for release:push * add release:publish * Adds release script helper (#535) * adds release helper function * update name and sep step for release:push * add release:publish * adds tagging to release:publish * fixes typo * added test to kmeans (#536) * docs: add jwilber as a contributor (#537) * docs: update README.md * docs: update .all-contributorsrc * docs: add tezzutezzu as a contributor (#538) * docs: update README.md * docs: update .all-contributorsrc * Update Contributing and Github docs (#539) * updated issue template: * update PR template * added new release how to * docs: add EmmaGoodliffe as a contributor (#545) * docs: update README.md * docs: update .all-contributorsrc * Update tfjs ~v1.2.2 (#544) * updated tfjs to 1.2.2 and bumped face-api to 0.20.1 * bump tensorflow models * bump body-pix version * adds version 2 to feature extractor * updated to mobilenetv2 api * updates to posenet 2 api * always process and resize images in mobilenet * update test with version 2 * update posenet inputs and test * fix issue with params in posenet * updates conditional in p5Utils * add loadImage test (#549) * Update npm helper scripts for making releases (#552) * updating release process scripts and docs * updated release proces npm scripts and docs * added release:sync and developement:sync" " * set fe to mobilenet v1 - use layers model (#554) * Adds Docsify to consolidate documentation (#556) * add docsify-cli as dev dep * adds docs structure for docsify * adds bodyPix documentation example * update bodypix documentation * adds imageClassifer docs * added styleguide template and updated docs * changed name of reference template * added dev guidelines * update dev guidelines * adds posenet docs * adds tutorials and faq * add dev:docs npm script * adds char rnn to docs * Adds documentation for references to docsify docs (#557) * add note on all contributors bot * adds faceapi header img" * adds design guidelines * adds design guidelines to sidebar * adds section for p5 web editor examples * adds cvae refs * update bodypix and charrnn docs * added dcgan docs * adds pitch detection docs * adds featureExtractor docs * adds knn classifier * update tutorial refs * adds pix2pix doc * adds sentiment docs * adds sketchrnn ref * adds sound classifier * adds styletransfer docs * adds unet docs * adds word2vec docs * adds yolo docs * adds acknowledgements section * adds acknowledgements section * Fixes Doodlenet - set new axes and .floor() (#559) * change split axis to 3 axes * use floor() instead of round() * adds options for detection confidence (#561) * Adds labels to custom image classification model from Teachable Machine output * Addressing issue #590 part 1 (#592) >The saved labels from Teachable Machine should be loaded by ml5. I believe this is working for sound classification, but not for images. Fetching labels from metadata.json when loading models from the new Teachable Machine. * docs: add EonYang as a contributor (#593) * docs: update README.md * docs: update .all-contributorsrc * docs: add lydiajessup as a contributor (#594) * docs: update README.md * docs: update .all-contributorsrc * set alias for sidebar (#598) * update docsify config * testing pathways * adds sublevel 2 * rm sidebar comment * testing sidebar nested list issue * relative path is false - using gh-pages * testing - pls ignore * testing plugin issue * drop alias * added no jekyll to root" * testing pathway for images * rm dot from path * set rel path * changed images to _media * use _media path for images * [Docs] Uses gh-pages to serve docsify docs (#599) * update docsify config * testing pathways * adds sublevel 2 * rm sidebar comment * testing sidebar nested list issue * relative path is false - using gh-pages * testing - pls ignore * testing plugin issue * drop alias * added no jekyll to root" * testing pathway for images * rm dot from path * set rel path * changed images to _media * use _media path for images * adds header nav to docs * updates alt text and image url * adds getting started url * adds reference index * updates href to reference in nav (#601) * [face-api] Removes expressions support from face-api in ml5.js (#602) * rm faceapi expressions support * rm expression ref in comment * updates tests for faceapi * fixes single detection and tests * Update docs (#603) * skeleton of generic Neural Network * proof of concept for training and prediction * use ml5 callcallbac to handle callback or promisek * added activation to options * load save model * adds placeholder to test and adds defaults * DIY Neural Net Updates (#568) * begins restructuring * adds tfjs-vis and adds generic createModel() function * adds tf viz and adds loadData function * adds normalization function * adds train function * adds predict function * work on DIY Neural Network (#569) * fixing output and hidden activations * moving data fields config to loadData() Eventually this should be ml5.utils() or ml5.data() * trimming whitespace * rename labels to targets for training output * using camelCase * moves shuffle out of normalize * [WIP] DIY Neural Net with focus on handling data internally (#571) * adds data obj in class * adds initialization step * creates nn data class * moves functions to data class * working with titanic * adds tf vis for training * changes tf viz show on training is debug is true * changes optimizer for regression * adds unNormalize function * fix normalization * rm log * refactors out to sep modules * adds options to train * adds options to train * adds comments * adds input/output array function * removes i/o label check * fixes optionOrCallback * fix test * adds comments * updates data handling * updates normalization data handling * DIY NN data handling updates (#572) * set up scaffold for creating inputs * added output scaffolding - turn into functions todo * adds function to encode data with ontHot or not * ensure inputTypes/outputTypes exist * [In progress] DIY NN - handles json and csv loading (#573) * adds placeholders for json and csv loader' * add json parsing * adds true flag to normalization to fix broken function * Fixes normalization function for NN (#575) * adds possible fix - moves norm to external function * adds correctly ordered data * DIY Neural Network: Handles onehot() encoding for inputs and outputs in normalization (#576) * adds correct input structure - accounts for onehot inputs * code formatting and cleaning * fixes need for onehot in classification for numeric output * adds whileTraining cb support to train (#577) * regression needs to pickup learning rate * Adds temporary fixes for i/o values given to/output by .predict() (#578) * adds min and max to meta and checks predict sample and forces array * adds onehot encoding legend * updating input handling for predict() * adds label,confidence output for classification * fix merge conflicts * rm log * need to use inputUnits and not input length * [WIP] Fixes for before class (#581) * changes for class * values not value, but this does not match feature extractor regression * [Diy nn] Fixes output reversal and gives unnormalized data as outputs of predict() (#582) * unnormalize outputs for regression * use .reverse() - note: we should use .unshift() where order matters * code cleanup * updates config inputs learning rate and code cleanup * changed value to return result.outputs * adds face-api and neural-network placeholder * adds section for web editor * set to /tree/development and adds posenet examples * adds link to src code * updates references to examples * sets tree/development as path * updates face-api and neural net docs * Revert "Update docs (#603)" (#604) This reverts commit c6b78d0a7aedc7050a095386ad88300d91a761b2. * Updates docsify docs (#605) * skeleton of generic Neural Network * proof of concept for training and prediction * use ml5 callcallbac to handle callback or promisek * added activation to options * load save model * adds placeholder to test and adds defaults * DIY Neural Net Updates (#568) * begins restructuring * adds tfjs-vis and adds generic createModel() function * adds tf viz and adds loadData function * adds normalization function * adds train function * adds predict function * work on DIY Neural Network (#569) * fixing output and hidden activations * moving data fields config to loadData() Eventually this should be ml5.utils() or ml5.data() * trimming whitespace * rename labels to targets for training output * using camelCase * moves shuffle out of normalize * [WIP] DIY Neural Net with focus on handling data internally (#571) * adds data obj in class * adds initialization step * creates nn data class * moves functions to data class * working with titanic * adds tf vis for training * changes tf viz show on training is debug is true * changes optimizer for regression * adds unNormalize function * fix normalization * rm log * refactors out to sep modules * adds options to train * adds options to train * adds comments * adds input/output array function * removes i/o label check * fixes optionOrCallback * fix test * adds comments * updates data handling * updates normalization data handling * DIY NN data handling updates (#572) * set up scaffold for creating inputs * added output scaffolding - turn into functions todo * adds function to encode data with ontHot or not * ensure inputTypes/outputTypes exist * [In progress] DIY NN - handles json and csv loading (#573) * adds placeholders for json and csv loader' * add json parsing * adds true flag to normalization to fix broken function * Fixes normalization function for NN (#575) * adds possible fix - moves norm to external function * adds correctly ordered data * DIY Neural Network: Handles onehot() encoding for inputs and outputs in normalization (#576) * adds correct input structure - accounts for onehot inputs * code formatting and cleaning * fixes need for onehot in classification for numeric output * adds whileTraining cb support to train (#577) * regression needs to pickup learning rate * Adds temporary fixes for i/o values given to/output by .predict() (#578) * adds min and max to meta and checks predict sample and forces array * adds onehot encoding legend * updating input handling for predict() * adds label,confidence output for classification * fix merge conflicts * rm log * need to use inputUnits and not input length * [WIP] Fixes for before class (#581) * changes for class * values not value, but this does not match feature extractor regression * [Diy nn] Fixes output reversal and gives unnormalized data as outputs of predict() (#582) * unnormalize outputs for regression * use .reverse() - note: we should use .unshift() where order matters * code cleanup * updates config inputs learning rate and code cleanup * changed value to return result.outputs * adds face-api and neural-network placeholder * adds section for web editor * set to /tree/development and adds posenet examples * adds link to src code * updates references to examples * sets tree/development as path * updates face-api and neural net docs * rm neural net class from update-docs branch * moves bodyPix to support preload() (#610) * Adds preload support to faceApi (#611) * moves bodyPix to support preload() * adds faceapi to preload * Updates Docs with latest features (#612) * adds face-api updates and neural network docs updates * adds docs for nn functions * adds save and load docs * adds neural Network documentation * fixes backticks * adds sidebar dividers * adds docs on model loading and contributors (#616) * Updates gh templates (#617) * updates PR template * updates gh issues template * updates template * Generic Neural Network class (#485) * skeleton of generic Neural Network * proof of concept for training and prediction * use ml5 callcallbac to handle callback or promisek * added activation to options * load save model * adds placeholder to test and adds defaults * DIY Neural Net Updates (#568) * begins restructuring * adds tfjs-vis and adds generic createModel() function * adds tf viz and adds loadData function * adds normalization function * adds train function * adds predict function * work on DIY Neural Network (#569) * fixing output and hidden activations * moving data fields config to loadData() Eventually this should be ml5.utils() or ml5.data() * trimming whitespace * rename labels to targets for training output * using camelCase * moves shuffle out of normalize * [WIP] DIY Neural Net with focus on handling data internally (#571) * adds data obj in class * adds initialization step * creates nn data class * moves functions to data class * working with titanic * adds tf vis for training * changes tf viz show on training is debug is true * changes optimizer for regression * adds unNormalize function * fix normalization * rm log * refactors out to sep modules * adds options to train * adds options to train * adds comments * adds input/output array function * removes i/o label check * fixes optionOrCallback * fix test * adds comments * updates data handling * updates normalization data handling * DIY NN data handling updates (#572) * set up scaffold for creating inputs * added output scaffolding - turn into functions todo * adds function to encode data with ontHot or not * ensure inputTypes/outputTypes exist * [In progress] DIY NN - handles json and csv loading (#573) * adds placeholders for json and csv loader' * add json parsing * adds true flag to normalization to fix broken function * Fixes normalization function for NN (#575) * adds possible fix - moves norm to external function * adds correctly ordered data * DIY Neural Network: Handles onehot() encoding for inputs and outputs in normalization (#576) * adds correct input structure - accounts for onehot inputs * code formatting and cleaning * fixes need for onehot in classification for numeric output * adds whileTraining cb support to train (#577) * regression needs to pickup learning rate * Adds temporary fixes for i/o values given to/output by .predict() (#578) * adds min and max to meta and checks predict sample and forces array * adds onehot encoding legend * updating input handling for predict() * adds label,confidence output for classification * fix merge conflicts * rm log * need to use inputUnits and not input length * [WIP] Fixes for before class (#581) * changes for class * values not value, but this does not match feature extractor regression * [Diy nn] Fixes output reversal and gives unnormalized data as outputs of predict() (#582) * unnormalize outputs for regression * use .reverse() - note: we should use .unshift() where order matters * code cleanup * updates config inputs learning rate and code cleanup * changed value to return result.outputs * adds blob reader for json * [diy nn] moves tf-vis visualizations to helper class (#583) * adds tfvis to neuralNetworkVis.js * adds tfvis to class * handle text and convert to json (#584) * allow user more control in the case of classification * allow user more control in the case of classification (#585) * Refactoring DIY Neural Network (#591) * [wip] refactoring data handling and logic separation * wip adding convertRawToTensor * adds conversion to tensor * adds normalization step * adds train() * adds predictInternal() * fixes unique value mapping issue * comment out .print() * return all regression results * adds experimental normalizationOptions object * quick fix for checking if normalizationOptions exists * commenting out auto tensor printing * adds checks for normalizationOptions * fix obj ref in conditional * adds .normalizeData() - keeps .normalize() for now * changes whileTraining function when debug is true * adds layer options for adding more than 2 layers - experimentalgit add . * remove .normalize() in favor of .normalizeData() * adds save() and load() * adds data.warmUp() to allow training without normalization - handled internally on train * updates outputs of .predict() and .classify() to be array * Divide by 0 issue when data is not normalized (#596) * skipping calculating inputMin and inputMax when data not normalized * adds check for this.data.meta.isNormalized in predictInternal * Diy nn code cleanup (#606) * adds input/output checks to specify number or array of input/output names * moves model creation to training to compile after input and output units have been calcd * cleanup code in .trainInternal() * move vars up to top of function in .trainInternal() * rm model creation from .createModelFromDataInternal() * code cleanup and adds comments to .initializeIOUnits() * code cleanup * [DIY NN] Adds .saveData() and .loadData() (#607) * adds saveData function * updates saveData() with input checks * adds loadData function * [Diy NN] Adds .predictMultiple() for batch predictions/classifications (#609) * adds predictMultiple() * rm ys.print() * moves bodypix and faceapi to preload support to match base branch * [DIY nn] Neural Net Fixes loading pre-trained model (#613) * adds model_meta.json to savedfiles * adds loading of model_meta.json - temp fix * use substring method for url checking * adds check for if input is JSON * loadData and text variables mixed up (#615) * [Diy nn] fix model loading load (Work in progress) (#618) * adds input object as param to .load() * adds fetch to retrieve data from urls in .load() based on object * updates object handler * fixes check for model.json * fixes parsing and ordering files * adds fix for p5 web editor - must use explicit json object to load models * allows user to add video to .segment() and .segmentWithParts() (#619) * updates unet docs (#620) * adds basic test for defaults (#621) * docs: add EmmaGoodliffe as a contributor (#622) * docs: update README.md * docs: update .all-contributorsrc * adds fix for unet image return for p5 (#625) * updates webpack dev server to 3.1.11 (#629) * Fixes vulnerabilities noted by npm (#630) * run npm audit fix * bumps commitizen dep * updated readme and readme updater (#632) * Update NeuralNetwork docs (#631) * skeleton of generic Neural Network * proof of concept for training and prediction * use ml5 callcallbac to handle callback or promisek * added activation to options * load save model * adds placeholder to test and adds defaults * DIY Neural Net Updates (#568) * begins restructuring * adds tfjs-vis and adds generic createModel() function * adds tf viz and adds loadData function * adds normalization function * adds train function * adds predict function * work on DIY Neural Network (#569) * fixing output and hidden activations * moving data fields config to loadData() Eventually this should be ml5.utils() or ml5.data() * trimming whitespace * rename labels to targets for training output * using camelCase * moves shuffle out of normalize * [WIP] DIY Neural Net with focus on handling data internally (#571) * adds data obj in class * adds initialization step * creates nn data class * moves functions to data class * working with titanic * adds tf vis for training * changes tf viz show on training is debug is true * changes optimizer for regression * adds unNormalize function * fix normalization * rm log * refactors out to sep modules * adds options to train * adds options to train * adds comments * adds input/output array function * removes i/o label check * fixes optionOrCallback * fix test * adds comments * updates data handling * updates normalization data handling * DIY NN data handling updates (#572) * set up scaffold for creating inputs * added output scaffolding - turn into functions todo * adds function to encode data with ontHot or not * ensure inputTypes/outputTypes exist * [In progress] DIY NN - handles json and csv loading (#573) * adds placeholders for json and csv loader' * add json parsing * adds true flag to normalization to fix broken function * Fixes normalization function for NN (#575) * adds possible fix - moves norm to external function * adds correctly ordered data * DIY Neural Network: Handles onehot() encoding for inputs and outputs in normalization (#576) * adds correct input structure - accounts for onehot inputs * code formatting and cleaning * fixes need for onehot in classification for numeric output * adds whileTraining cb support to train (#577) * regression needs to pickup learning rate * Adds temporary fixes for i/o values given to/output by .predict() (#578) * adds min and max to meta and checks predict sample and forces array * adds onehot encoding legend * updating input handling for predict() * adds label,confidence output for classification * fix merge conflicts * rm log * need to use inputUnits and not input length * [WIP] Fixes for before class (#581) * changes for class … * adds canvas support for input handling in UNET (#700) * updates docs and bumps version (#701) --- README.md | 8 +++++--- docs/README.md | 8 ++++---- docs/tutorials/hello-ml5.md | 2 +- package.json | 2 +- src/UNET/index.js | 2 ++ 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3c79579d1..d70aa2f27 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,14 @@ There are several ways you can use the ml5.js library: +

-* You can use the latest version (0.4.2) by adding it to the head section of your HTML document: +* You can use the latest version (0.4.3) by adding it to the head section of your HTML document: -**v0.4.2** +**v0.4.3** - +

@@ -43,6 +44,7 @@ There are several ways you can use the ml5.js library: + * If you need to use an earlier version for any reason, you can change the version number. The [previous versions of ml5 can be found here](https://www.npmjs.com/package/ml5). You can use those previous versions by replacing `` with the ml5 version of interest: diff --git a/docs/README.md b/docs/README.md index 7d31404b7..1f7c5c22e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,13 +19,13 @@ The fastest way to get started exploring the creative possibilities of ml5.js ar 3. You can also copy and paste the cdn link to the ml5 library here: ``` - + ``` *** #### Quickstart: Plain JavaScript -Reference the [latest version](https://unpkg.com/ml5@0.4.2/dist/ml5.min.js) of ml5.js using a script tag in an HTML file as below: +Reference the [latest version](https://unpkg.com/ml5@0.4.3/dist/ml5.min.js) of ml5.js using a script tag in an HTML file as below: In an **index.html** file, copy and paste the following and open up that file in your web browser. @@ -36,7 +36,7 @@ In an **index.html** file, copy and paste the following and open up that file in Getting Started with ml5.js - + @@ -71,7 +71,7 @@ In an **index.html** file, copy and paste the following and open up that file in - + diff --git a/docs/tutorials/hello-ml5.md b/docs/tutorials/hello-ml5.md index 94e9d51ca..603415d44 100644 --- a/docs/tutorials/hello-ml5.md +++ b/docs/tutorials/hello-ml5.md @@ -72,7 +72,7 @@ Here you can see that we read in the javascript libraries. This includes our ml5 - + diff --git a/package.json b/package.json index 7aaca7f2a..5896b91c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ml5", - "version": "0.4.2", + "version": "0.4.3", "description": "A friendly machine learning library for the web.", "main": "dist/ml5.min.js", "directories": { diff --git a/src/UNET/index.js b/src/UNET/index.js index 0afc38690..7905b9583 100644 --- a/src/UNET/index.js +++ b/src/UNET/index.js @@ -53,10 +53,12 @@ class UNET { if (inputOrCallback instanceof HTMLImageElement || inputOrCallback instanceof HTMLVideoElement || + inputOrCallback instanceof HTMLCanvasElement || inputOrCallback instanceof ImageData) { imgToPredict = inputOrCallback; } else if (typeof inputOrCallback === 'object' && (inputOrCallback.elt instanceof HTMLImageElement || inputOrCallback.elt instanceof HTMLVideoElement || + inputOrCallback.elt instanceof HTMLCanvasElement || inputOrCallback.elt instanceof ImageData)) { imgToPredict = inputOrCallback.elt; } else if (typeof inputOrCallback === 'function') { From 2fc5db006a29c55c57a4625ed60ba2dfcff3a8f1 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 15 Nov 2019 19:53:32 -0500 Subject: [PATCH 23/25] Add support for offline models --- src/ObjectDetector/index.js | 53 +++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index d09e9ed85..f363e3ed0 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -20,48 +20,61 @@ class ObjectDetector { */ /** * Create ObjectDetector model. Works on video and images. - * @param {string} modelName - The name or the URL of the model to use. Current model name options - * are: 'YOLO'. - * @param {HTMLVideoElement} video - The video to be used for object detection and classification. + * @param {string} modelNameOrUrl - The name or the URL of the model to use. Current model name options + * are: 'YOLO' and 'CocoSsd'. * @param {Object} options - Optional. A set of options. * @param {function} callback - Optional. A callback function that is called once the model has loaded. * If no callback is provided, it will return a promise that will be resolved once the model has loaded. */ - constructor(modelName, video, options, callback) { - this.modelName = modelName; + constructor(modelNameOrUrl, video, options, callback) { + this.modelNameOrUrl = modelNameOrUrl; this.video = video; this.options = options || {}; this.callback = callback; if (isInstanceOfSupportedElement(video)) { this.video = video; - } else if (typeof video === 'object' && isInstanceOfSupportedElement(video.elt)) { + } else if ( + typeof video === "object" && + isInstanceOfSupportedElement(video.elt) + ) { this.video = video.elt; // Handle p5.js video and image } - switch (modelName) { - case 'YOLO': - this.model = new YOLO({ disableDeprecationNotice: true, ...options }, callback); + switch (modelNameOrUrl) { + case "YOLO": + this.model = new YOLO( + { disableDeprecationNotice: true, ...options }, + callback + ); break; - case 'CocoSsd': + case "CocoSsd": this.model = new CocoSsd(callback); break; default: - throw new Error('Model name not supported') + // Uses custom model url + this.model = new YOLO( + { + disableDeprecationNotice: true, + modelUrl: modelNameOrUrl, + ...options + }, + callback + ); } } /** - * @typedef {Object} ObjectDetectorPrediction - * @property {number} x - top left x coordinate of the prediction box (0 to 1). - * @property {number} y - top left y coordinate of the prediction box (0 to 1). - * @property {number} w - width of the prediction box (0 to 1). - * @property {number} h - height of the prediction box (0 to 1). - * @property {string} label - the label given. - * @property {number} confidence - the confidence score (0 to 1). - */ + * @typedef {Object} ObjectDetectorPrediction + * @property {number} x - top left x coordinate of the prediction box (0 to 1). + * @property {number} y - top left y coordinate of the prediction box (0 to 1). + * @property {number} w - width of the prediction box (0 to 1). + * @property {number} h - height of the prediction box (0 to 1). + * @property {string} label - the label given. + * @property {number} confidence - the confidence score (0 to 1). + */ /** - * Returns an rgb array + * Returns an array of predicted objects * @param {function} callback - Optional. A callback that deliver the result. If no callback is * given, a promise is will be returned. * @return {ObjectDetectorPrediction[]} an array of the prediction result From f42a1f16221dd81bc41c0ff8fd56f85c3e08a120 Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 15 Nov 2019 20:45:43 -0500 Subject: [PATCH 24/25] Support offline models for object detection and remove detection subject from constructor --- src/ObjectDetector/YOLO/index.js | 27 +++++++++++++++------------ src/ObjectDetector/index.js | 22 ++++++++-------------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/ObjectDetector/YOLO/index.js b/src/ObjectDetector/YOLO/index.js index bcfcc6887..09bdc1789 100644 --- a/src/ObjectDetector/YOLO/index.js +++ b/src/ObjectDetector/YOLO/index.js @@ -11,7 +11,10 @@ Heavily derived from https://github.com/ModelDepot/tfjs-yolo-tiny (ModelDepot: m import * as tf from '@tensorflow/tfjs'; import Video from './../../utils/Video'; -import { imgToTensor } from './../../utils/imageUtilities'; +import { + imgToTensor, + isInstanceOfSupportedElement +} from "./../../utils/imageUtilities"; import callCallback from './../../utils/callcallback'; import CLASS_NAMES from './../../utils/COCO_CLASSES'; import modelLoader from './../../utils/modelLoader'; @@ -46,10 +49,9 @@ class YOLOBase extends Video { */ /** * Create YOLO model. Works on video and images. - * @param {HTMLVideoElement} video - Optional. The video to be used for object detection and classification. + * @param {HTMLVideoElement|HTMLImageElement|HTMLCanvasElement|ImageData} video - Optional. The video to be used for object detection and classification. * @param {Object} options - Optional. A set of options. - * @param {function} callback - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise - * that will be resolved once the model has loaded. + * @param {function} callback - Optional. A callback function that is called once the model has loaded. */ constructor(video, options, callback) { super(video, imageSize); @@ -84,21 +86,22 @@ class YOLOBase extends Video { return this; } + /** + * Detect objects that are in video, returns bounding box, label, and confidence scores + * @param {HTMLVideoElement|HTMLImageElement|HTMLCanvasElement|ImageData} inputOrCallback - Subject of the detection, or callback + * @param {function} cb - Optional. A callback function that is called once the model has loaded. If no callback is provided, it will return a promise + * that will be resolved once the prediction is done. + */ async detect(inputOrCallback, cb) { await this.ready; let imgToPredict; let callback = cb; - if (inputOrCallback instanceof HTMLImageElement - || inputOrCallback instanceof HTMLVideoElement - || inputOrCallback instanceof HTMLCanvasElement - || inputOrCallback instanceof ImageData) { + if (isInstanceOfSupportedElement(inputOrCallback)) { imgToPredict = inputOrCallback; - } else if (typeof inputOrCallback === 'object' && (inputOrCallback.elt instanceof HTMLImageElement - || inputOrCallback.elt instanceof HTMLVideoElement - || inputOrCallback.elt instanceof ImageData)) { + } else if (typeof inputOrCallback === "object" && isInstanceOfSupportedElement(inputOrCallback.elt)) { imgToPredict = inputOrCallback.elt; // Handle p5.js image and video. - } else if (typeof inputOrCallback === 'function') { + } else if (typeof inputOrCallback === "function") { imgToPredict = this.video; callback = inputOrCallback; } diff --git a/src/ObjectDetector/index.js b/src/ObjectDetector/index.js index f363e3ed0..f95dc8353 100644 --- a/src/ObjectDetector/index.js +++ b/src/ObjectDetector/index.js @@ -24,23 +24,12 @@ class ObjectDetector { * are: 'YOLO' and 'CocoSsd'. * @param {Object} options - Optional. A set of options. * @param {function} callback - Optional. A callback function that is called once the model has loaded. - * If no callback is provided, it will return a promise that will be resolved once the model has loaded. */ - constructor(modelNameOrUrl, video, options, callback) { + constructor(modelNameOrUrl, options, callback) { this.modelNameOrUrl = modelNameOrUrl; - this.video = video; this.options = options || {}; this.callback = callback; - if (isInstanceOfSupportedElement(video)) { - this.video = video; - } else if ( - typeof video === "object" && - isInstanceOfSupportedElement(video.elt) - ) { - this.video = video.elt; // Handle p5.js video and image - } - switch (modelNameOrUrl) { case "YOLO": this.model = new YOLO( @@ -79,8 +68,13 @@ class ObjectDetector { * given, a promise is will be returned. * @return {ObjectDetectorPrediction[]} an array of the prediction result */ - detect(callback) { - return this.model.detect(this.video, callback); + detect(subject, callback) { + if (isInstanceOfSupportedElement(subject)) { + return this.model.detect(subject, callback); + } else if (typeof subject === "object" && isInstanceOfSupportedElement(subject.elt)) { + return this.model.detect(subject.elt, callback); // Handle p5.js video and image + } + throw new Error('Detection subject not supported'); } } From e1b9346e90ff307658bdeb8c56fc40616b43ae0e Mon Sep 17 00:00:00 2001 From: Tirta Rachman Date: Fri, 15 Nov 2019 22:30:29 -0500 Subject: [PATCH 25/25] Add some feature tests --- karma.conf.js | 1 + src/ObjectDetector/CocoSsd/index_test.js | 49 ++++++++++++++++++++++++ src/ObjectDetector/YOLO/index_test.js | 2 +- src/ObjectDetector/index_test.js | 26 +++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 src/ObjectDetector/CocoSsd/index_test.js create mode 100644 src/ObjectDetector/index_test.js diff --git a/karma.conf.js b/karma.conf.js index e7e918f95..8a791e483 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -8,6 +8,7 @@ module.exports = (config) => { files: [ 'src/index.js', `src/${config.model ? config.model : '**'}/*_test.js`, + `src/${config.model ? config.model : '**'}/**/*_test.js`, ], preprocessors: { 'src/index.js': ['webpack'], diff --git a/src/ObjectDetector/CocoSsd/index_test.js b/src/ObjectDetector/CocoSsd/index_test.js new file mode 100644 index 000000000..9340ce400 --- /dev/null +++ b/src/ObjectDetector/CocoSsd/index_test.js @@ -0,0 +1,49 @@ +// Copyright (c) 2019 ml5 +// +// This software is released under the MIT License. +// https://opensource.org/licenses/MIT + +describe('CocoSsd', () => { + let cocoSsd; + + async function getRobin() { + const img = new Image(); + img.crossOrigin = ''; + img.src = 'https://cdn.jsdelivr.net/gh/ml5js/ml5-library@development/assets/bird.jpg'; + await new Promise((resolve) => { img.onload = resolve; }); + return img; + } + + async function getImageData() { + const arr = new Uint8ClampedArray(40000); + + // Iterate through every pixel + for (let i = 0; i < arr.length; i += 4) { + arr[i + 0] = 0; // R value + arr[i + 1] = 190; // G value + arr[i + 2] = 0; // B value + arr[i + 3] = 255; // A value + } + + // Initialize a new ImageData object + const img = new ImageData(arr, 200); + return img; + } + + beforeEach(async () => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = 100000; + cocoSsd = await objectDetector('CocoSsd'); + }); + + it('detects a robin', async () => { + const robin = await getRobin(); + const detection = await cocoSsd.detect(robin); + expect(detection[0].label).toBe('bird'); + }); + + it('detects takes ImageData', async () => { + const img = await getImageData(); + const detection = await cocoSsd.detect(img); + expect(detection).toEqual([]); + }); +}); diff --git a/src/ObjectDetector/YOLO/index_test.js b/src/ObjectDetector/YOLO/index_test.js index 56293b2fc..4c7d69f20 100644 --- a/src/ObjectDetector/YOLO/index_test.js +++ b/src/ObjectDetector/YOLO/index_test.js @@ -41,7 +41,7 @@ describe('YOLO', () => { beforeEach(async () => { jasmine.DEFAULT_TIMEOUT_INTERVAL = 100000; - yolo = await YOLO(); + yolo = await YOLO({ disableDeprecationNotice: true }); }); it('instantiates the YOLO classifier with defaults', () => { diff --git a/src/ObjectDetector/index_test.js b/src/ObjectDetector/index_test.js new file mode 100644 index 000000000..d84d5b9cf --- /dev/null +++ b/src/ObjectDetector/index_test.js @@ -0,0 +1,26 @@ +// Copyright (c) 2019 ml5 +// +// This software is released under the MIT License. +// https://opensource.org/licenses/MIT + +const { objectDetector } = ml5; + +xdescribe('ObjectDetector', () => { + let cocoSsd; + + beforeEach(async () => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = 100000; + cocoSsd = await objectDetector('CocoSsd'); + }); + + it('throws error when a non image is trying to be detected', async () => { + const notAnImage = 'not_an_image' + try { + await cocoSsd.detect(notAnImage); + fail('Error should have been thrown'); + } + catch (error) { + expect(error.message).toBe('Detection subject not supported'); + } + }); +});