Add object detection API for AutoML

tfjs-automl/README.md

@@ -22,7 +22,7 @@ If you are using CDN:
 
 We support the following types of AutoML Edge models:
 1) [Image classification](#image-classification)
-2) **[In progress]** [Object detection](#object-detection)
+2) [Object detection](#object-detection)
 
 ## Image classification
 
@@ -67,7 +67,7 @@ a 3D [`Tensor`](https://js.tensorflow.org/api/latest/#class:Tensor):
 
 ```js
 const img = document.getElementById('img');
-const options = {};
+const options = {centerCrop: true};
 const predictions = await model.classify(img, options);
 ```
 
@@ -88,6 +88,90 @@ probabilities:
 ]
 ```
 
+### Advanced usage
+
+Advanced users can access the underlying
+[`GraphModel`](https://js.tensorflow.org/api/latest/#class:GraphModel) via
+`model.graphModel`. The `GraphModel` allows users to call lower level methods
+such as `predict()`, `execute()` and `executeAsync()` which return tensors.
+
+`model.dictionary` gives you access to the ordered list of labels.
+
 ## Object detection
 
-TODO(smilkov): Write this when object detection is ready.
+AutoML Object detection model will output the following set of files:
+- `model.json`, the model topology
+- `dict.txt`, a newline-separated list of labels
+- One or more of `*.bin` files which hold the weights
+
+Make sure you can access those files as static assets from your web app by serving them locally or on Google Cloud Storage.
+
+### Demo
+
+The object detection demo lives in
+[demo/object_classification](./demo/object_classification). To run it:
+
+```sh
+cd demo/object_detection
+yarn
+yarn watch
+```
+
+This will start a local HTTP server on port 1234 that serves the demo.
+
+### Loading the model
+```js
+import * as automl from '@tensorflow/tfjs-automl';
+const modelUrl = 'model.json'; // URL to the model.json file.
+const model = await automl.loadObjectDetection(modelUrl);
+```
+
+### Making a prediction
+The input `img` can be
+[`HTMLImageElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLImageElement),
+[`HTMLCanvasElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement),
+[`HTMLVideoElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement),
+[`ImageData`](https://developer.mozilla.org/en-US/docs/Web/API/ImageData) or
+a 3D [`Tensor`](https://js.tensorflow.org/api/latest/#class:Tensor):
+
+```html
+<img id="img" src="PATH_TO_IMAGE" />
+```
+
+```js
+const img = document.getElementById('img');
+const options = {score: 0.5, iou: 0.5, topk: 20};
+const predictions = await model.detect(img, options);
+```
+
+`options` is optional and has the following properties:
+- `score` - Probability score between 0 and 1. Defaults to 0.5. Boxes with score lower than this threshold will be ignored.
+- `topk` - Only the `topk` most likely objects are returned. The actual number of objects might be less than this number.
+- `iou` - Intersection over union threshold. IoU is a metric between 0 and 1 used to measure the overlap of two boxes. The predicted boxes will not overlap more than the specified threshold.
+
+The result `predictions` is a sorted list of predicted objects:
+
+```js
+[
+  {
+    box: {
+      left: 105.1,
+      top: 22.2,
+      width: 70.6,
+      height: 55.7
+    },
+    label: "Tomato",
+    score: 0.972
+  },
+  ...
+]
+```
+
+### Advanced usage
+
+Advanced users can access the underlying
+[`GraphModel`](https://js.tensorflow.org/api/latest/#class:GraphModel) via
+`model.graphModel`. The `GraphModel` allows users to call lower level methods
+such as `predict()`, `execute()` and `executeAsync()` which return tensors.
+
+`model.dictionary` gives you access to the ordered list of labels.

tfjs-automl/demo/img_classification/package.json

@@ -1,8 +1,8 @@
 {
-  "name": "automl-demo",
+  "name": "automl-img-classification-demo",
   "version": "0.0.1",
   "private": true,
-  "description": "Demo of using automl client",
+  "description": "Image classification demo using the AutoML NPM library",
   "main": "index.js",
   "scripts": {
     "watch": "cross-env NODE_ENV=development parcel index.html --no-hmr --open ",
@@ -18,5 +18,9 @@
     "parcel-bundler": "~1.10.3",
     "yalc": "~1.0.0-pre.27"
   },
+  "dependencies": {
+    "@tensorflow/tfjs-converter": "^1.2.8",
+    "@tensorflow/tfjs-core": "^1.2.8"
+  },
   "license": "Apache-2.0"
 }

tfjs-automl/demo/img_classification/yarn.lock

@@ -697,11 +697,48 @@
   resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b"
   integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==
 
+"@tensorflow/tfjs-converter@^1.2.8":
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-converter/-/tfjs-converter-1.2.8.tgz#86fa47be3e92a90d4191956f08015a17b93c3ef9"
+  integrity sha512-weHzkNRVgnY9TcbA3XTneNgCyuIXLjF8ks8YbFA+81i2w6qO90xiAdWtP2YmR+F9K9S4WR3bSSB0AQKZAp+mPQ==
+
+"@tensorflow/tfjs-core@^1.2.8":
+  version "1.2.8"
+  resolved "https://registry.yarnpkg.com/@tensorflow/tfjs-core/-/tfjs-core-1.2.8.tgz#d6873b88522f8cf25d34c10afd095866578d7d92"
+  integrity sha512-lWV4vAnXAAmahXpCWBwdGGW9HO6iNw9pUeVYih7pDXeJahMk3OJs6SgjRNhwn+ldsGwRoorR0/RHg0yNLmqWxQ==
+  dependencies:
+    "@types/offscreencanvas" "~2019.3.0"
+    "@types/seedrandom" "2.4.27"
+    "@types/webgl-ext" "0.0.30"
+    "@types/webgl2" "0.0.4"
+    node-fetch "~2.1.2"
+    seedrandom "2.4.3"
+
+"@types/offscreencanvas@~2019.3.0":
+  version "2019.3.0"
+  resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz#3336428ec7e9180cf4566dfea5da04eb586a6553"
+  integrity sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==
+
 "@types/q@^1.5.1":
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/@types/q/-/q-1.5.2.tgz#690a1475b84f2a884fd07cd797c00f5f31356ea8"
   integrity sha512-ce5d3q03Ex0sy4R14722Rmt6MT07Ua+k4FwDfdcToYJcMKNtRVQvJ6JCAPdAmAnbRb6CsX6aYb9m96NGod9uTw==
 
+"@types/seedrandom@2.4.27":
+  version "2.4.27"
+  resolved "https://registry.yarnpkg.com/@types/seedrandom/-/seedrandom-2.4.27.tgz#9db563937dd86915f69092bc43259d2f48578e41"
+  integrity sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE=
+
+"@types/webgl-ext@0.0.30":
+  version "0.0.30"
+  resolved "https://registry.yarnpkg.com/@types/webgl-ext/-/webgl-ext-0.0.30.tgz#0ce498c16a41a23d15289e0b844d945b25f0fb9d"
+  integrity sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==
+
+"@types/webgl2@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@types/webgl2/-/webgl2-0.0.4.tgz#c3b0f9d6b465c66138e84e64cb3bdf8373c2c279"
+  integrity sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==
+
 abbrev@1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@@ -3688,6 +3725,11 @@ node-addon-api@^1.6.0:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-1.7.1.tgz#cf813cd69bb8d9100f6bdca6755fc268f54ac492"
   integrity sha512-2+DuKodWvwRTrCfKOeR24KIc5unKjOh8mz17NCzVnHWfjAdDqbfbjqh7gUT+BkXBRQM52+xCHciKWonJ3CbJMQ==
 
+node-fetch@~2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5"
+  integrity sha1-q4hOjn5X44qUR1POxwb3iNF2i7U=
+
 node-forge@^0.7.1:
   version "0.7.6"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.7.6.tgz#fdf3b418aee1f94f0ef642cd63486c77ca9724ac"
@@ -5121,6 +5163,11 @@ sax@^1.2.4, sax@~1.2.1, sax@~1.2.4:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
+seedrandom@2.4.3:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/seedrandom/-/seedrandom-2.4.3.tgz#2438504dad33917314bff18ac4d794f16d6aaecc"
+  integrity sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw=
+
 "semver@2 || 3 || 4 || 5", semver@^5.3.0, semver@^5.4.1, semver@^5.5.0, semver@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"

tfjs-automl/demo/object_detection/.babelrc

@@ -0,0 +1,16 @@
+{
+  "presets": [
+    [
+      "env",
+      {
+        "esmodules": false,
+        "targets": {
+          "browsers": [
+            "> 3%"
+          ]
+        }
+      }
+    ]
+  ],
+  "plugins": []
+}

tfjs-automl/demo/object_detection/index.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>AutoML Object detection demo</title>
+  <style>
+  </style>
+</head>
+<body>
+  <div id="imgDiv" style="position:relative;">
+    <img id="salad" width="500" crossorigin="anonymous" src="https://storage.googleapis.com/tfjs-testing/tfjs-automl/object_detection/test_image.jpg" />
+    <svg width="500" height="375" style="position: absolute;top:0;left:0;">
+      <style>
+        .box {
+          stroke-width: 2;
+          fill: none;
+          stroke: red;
+        }
+        .label {
+          font-size: 12px;
+          fill: white;
+          text-anchor: middle;
+        }
+        .label-rect {
+          fill: black;
+        }
+      </style>
+    </svg>
+  </div>
+  <script src="index.js"></script>
+</body>
+</html>

tfjs-automl/demo/object_detection/index.js

@@ -0,0 +1,67 @@
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+// TODO(smilkov): Import from "@tensoflow/tfjs-automl" when the package
+// is released.
+import * as automl from '../../src/index';
+
+const MODEL_URL =
+    'https://storage.googleapis.com/tfjs-testing/tfjs-automl/object_detection/model.json';
+
+async function run() {
+  const model = await automl.loadObjectDetection(MODEL_URL);
+  const image = document.getElementById('salad');
+  // These are the default options.
+  const options = {score: 0.5, iou: 0.5, topk: 20};
+  const predictions = await model.detect(image, options);
+  drawBoxes(predictions);
+}
+
+// Overlays boxes with labels onto the image using `rect` and `text` svg
+// elements.
+function drawBoxes(predictions) {
+  const svg = document.querySelector('svg');
+  predictions.forEach(prediction => {
+    const {box, label, score} = prediction;
+    const {left, top, width, height} = box;
+    const rect = document.createElementNS('http://www.w3.org/2000/svg', 'rect');
+    rect.setAttribute('width', width);
+    rect.setAttribute('height', height);
+    rect.setAttribute('x', left);
+    rect.setAttribute('y', top);
+    rect.setAttribute('class', 'box');
+    const text = document.createElementNS('http://www.w3.org/2000/svg', 'text');
+    text.setAttribute('x', left + width / 2);
+    text.setAttribute('y', top);
+    text.setAttribute('dy', 12);
+    text.setAttribute('class', 'label');
+    text.textContent = `${label}: ${score.toFixed(3)}`;
+    svg.appendChild(rect);
+    svg.appendChild(text);
+    const textBBox = text.getBBox();
+    const textRect =
+        document.createElementNS('http://www.w3.org/2000/svg', 'rect');
+    textRect.setAttribute('x', textBBox.x);
+    textRect.setAttribute('y', textBBox.y);
+    textRect.setAttribute('width', textBBox.width);
+    textRect.setAttribute('height', textBBox.height);
+    textRect.setAttribute('class', 'label-rect');
+    svg.insertBefore(textRect, text);
+  });
+}
+
+run();

tfjs-automl/demo/object_detection/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "automl-object-detection-demo",
+  "version": "0.0.1",
+  "private": true,
+  "description": "Object detection demo using the AutoML NPM library",
+  "main": "index.js",
+  "scripts": {
+    "watch": "cross-env NODE_ENV=development parcel index.html --no-hmr --open ",
+    "build": "cross-env NODE_ENV=production parcel build index.html --public-url ./"
+  },
+  "devDependencies": {
+    "babel-core": "^6.26.3",
+    "babel-plugin-transform-runtime": "~6.23.0",
+    "babel-polyfill": "~6.26.0",
+    "babel-preset-env": "~1.6.1",
+    "clang-format": "~1.2.2",
+    "cross-env": "^5.2.0",
+    "parcel-bundler": "~1.10.3",
+    "yalc": "~1.0.0-pre.27"
+  },
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@tensorflow/tfjs-converter": "^1.2.8",
+    "@tensorflow/tfjs-core": "^1.2.8"
+  }
+}