feat: add teachable machine with custom trained model

indexai.html

@@ -0,0 +1,61 @@
+<div>Teachable Machine Image Model</div>
+<button type="button" onclick="init()">Start</button>
+<div id="webcam-container"></div>
+<div id="label-container"></div>
+<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@latest/dist/tf.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/@teachablemachine/image@latest/dist/teachablemachine-image.min.js"></script>
+<script type="text/javascript">
+    // More API functions here:
+    // https://github.com/googlecreativelab/teachablemachine-community/tree/master/libraries/image
+
+    // the link to your model provided by Teachable Machine export panel
+    const URL = "https://192.168.1.39:7000/my-model/";
+
+    let model, webcam, labelContainer, maxPredictions;
+
+    // Load the image model and setup the webcam
+    async function init() {
+        const modelURL = URL + "model.json";
+        const metadataURL = URL + "metadata.json";
+
+        // load the model and metadata
+        // Refer to tmImage.loadFromFiles() in the API to support files from a file picker
+        // or files from your local hard drive
+        // Note: the pose library adds "tmImage" object to your window (window.tmImage)
+        model = await tmImage.load(modelURL, metadataURL);
+        maxPredictions = model.getTotalClasses();
+
+        // Convenience function to setup a webcam
+        const flip = true; // whether to flip the webcam
+        webcam = new tmImage.Webcam(200, 200, flip); // width, height, flip
+        await webcam.setup(); // request access to the webcam
+        await webcam.play();
+        window.requestAnimationFrame(loop);
+
+        // append elements to the DOM
+        document.getElementById("webcam-container").appendChild(webcam.canvas);
+        labelContainer = document.getElementById("label-container");
+        for (let i = 0; i < maxPredictions; i++) { // and class labels
+            labelContainer.appendChild(document.createElement("div"));
+        }
+    }
+
+
+    async function loop() {
+        webcam.update(); // update the webcam frame
+        await predict();
+        window.requestAnimationFrame(loop);
+    }
+
+    // run the webcam image through the image model
+    async function predict() {
+        // predict can take in an image, video or canvas html element
+        const prediction = await model.predict(webcam.canvas);
+        console.log(prediction);
+        for (let i = 0; i < maxPredictions; i++) {
+            const classPrediction =
+                prediction[i].className + ": " + prediction[i].probability.toFixed(2);
+            labelContainer.childNodes[i].innerHTML = classPrediction;
+        }
+    }
+</script>

models/tm--image/metadata.json

@@ -0,0 +1,15 @@
+{
+    "tfjsVersion": "1.7.4",
+    "tmVersion": "2.4.10",
+    "packageVersion": "0.8.4-alpha2",
+    "packageName": "@teachablemachine/image",
+    "timeStamp": "2025-11-15T13:33:10.951Z",
+    "userMetadata": {},
+    "modelName": "tm-my-image-model",
+    "labels": [
+        "ok",
+        "stop",
+        "unknown"
+    ],
+    "imageSize": 224
+}

python/main.py

@@ -1,8 +1,10 @@
 from arduino.app_utils import App, Bridge
 from arduino.app_bricks.web_ui import WebUI
 from arduino.app_bricks.object_detection import ObjectDetection
+from fastapi.responses import FileResponse
 import time
 import base64
+import os
 
 object_detection = ObjectDetection()
 
@@ -88,4 +90,22 @@ def on_modulino_button_pressed(btn):
 
 Bridge.provide("modulino_button_pressed", on_modulino_button_pressed)
 
+
+def serve_model_file(filepath):
+    """Serve model files with CORS headers"""
+    return FileResponse(
+        os.path.join("/app/assets/models/tm-my-image-model", filepath),
+        headers={
+            "Cache-Control": "no-store",
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
+            "Access-Control-Allow-Headers": "*",
+        },
+    )
+
+
+ui.expose_api("GET", "/my-model/model.json", lambda: serve_model_file("model.json"))
+ui.expose_api("GET", "/my-model/metadata.json", lambda: serve_model_file("metadata.json"))
+ui.expose_api("GET", "/my-model/weights.bin", lambda: serve_model_file("weights.bin"))
+
 App.run()

scratch-arduino-extensions/packages/scratch-vm/src/extensions/tmachine_image/index.js

@@ -0,0 +1,205 @@
+const BlockType = require("../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/block-type");
+const ArgumentType = require(
+  "../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/argument-type",
+);
+
+const Video = require("../../../../../../scratch-editor/packages/scratch-vm/src/io/video");
+
+// TODO add icons
+const iconURI = "";
+const menuIconURI = "";
+
+class TeachableMachineImage {
+  constructor(runtime) {
+    this.runtime = runtime;
+
+    this.modelLabels = [];
+    this.model = null;
+    this.predictions = [];
+    this.isModelLoaded = false;
+    this.modelURL = "https://192.168.1.39:7000/my-model/";
+
+    this.fetchModelLabels();
+    this.loadModel();
+  }
+
+  async fetchModelLabels() {
+    try {
+      const response = await fetch(this.modelURL + "/metadata.json");
+      const metadata = await response.json();
+      this.modelLabels = metadata.labels || [];
+      console.log("Fetched model labels:", this.modelLabels);
+    } catch (error) {
+      console.error("Error fetching model labels:", error);
+      this.modelLabels = []; // fallback
+    }
+  }
+
+  async loadModel() {
+    try {
+      // Load TensorFlow.js and Teachable Machine library
+      if (!window.tf) {
+        await this.loadScript("https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@latest/dist/tf.min.js");
+      }
+      if (!window.tmImage) {
+        await this.loadScript(
+          "https://cdn.jsdelivr.net/npm/@teachablemachine/image@latest/dist/teachablemachine-image.min.js",
+        );
+      }
+
+      const modelURL = this.modelURL + "model.json";
+      const metadataURL = this.modelURL + "metadata.json";
+
+      this.model = await tmImage.load(modelURL, metadataURL);
+      this.isModelLoaded = true;
+      console.log("Teachable Machine model loaded successfully");
+    } catch (error) {
+      console.error("Error loading Teachable Machine model:", error);
+    }
+  }
+
+  loadScript(src) {
+    return new Promise((resolve, reject) => {
+      const script = document.createElement("script");
+      script.src = src;
+      script.onload = resolve;
+      script.onerror = reject;
+      document.head.appendChild(script);
+    });
+  }
+
+  startPredictionLoop() {
+    if (!this.isModelLoaded) {
+      console.log("Model not loaded");
+      return;
+    }
+
+    const predict = async () => {
+      try {
+        const canvas = this.runtime.ioDevices.video.getFrame({
+          format: Video.FORMAT_CANVAS,
+          dimensions: [480, 360], // the same as the stage resolution
+        });
+        if (!canvas) {
+          console.log("No canvas available from video frame.");
+          return;
+        }
+        const prediction = await this.model.predict(canvas);
+        console.log("preditions", prediction);
+        this.predictions = prediction;
+      } catch (error) {
+        console.error("Prediction error:", error);
+      }
+
+      // Continue loop
+      setTimeout(predict, 100); // Predict every 100ms
+    };
+
+    predict();
+  }
+
+  getConfidence(className) {
+    if (!this.predictions) return 0;
+    const prediction = this.predictions.find(p => p.className === className);
+    return prediction ? prediction.probability : 0;
+  }
+}
+
+TeachableMachineImage.prototype.getInfo = function() {
+  return {
+    id: "TeachableMachineImage",
+    name: "Teachable Machine Image",
+    menuIconURI: menuIconURI,
+    blockIconURI: iconURI,
+    blocks: [
+      {
+        opcode: "startDetectionLoop",
+        blockType: BlockType.COMMAND,
+        text: "start detection",
+        func: "startDetectionLoop",
+        arguments: {},
+      },
+      {
+        opcode: "whenObjectDetected",
+        blockType: BlockType.HAT,
+        text: "when [OBJECT] detected with confidence > [THRESHOLD]%",
+        func: "whenObjectDetected",
+        arguments: {
+          OBJECT: {
+            type: ArgumentType.STRING,
+            menu: "modelLabels",
+            defaultValue: "ok",
+          },
+          THRESHOLD: {
+            type: ArgumentType.NUMBER,
+            defaultValue: 50,
+          },
+        },
+      },
+      {
+        opcode: "isObjectDetected",
+        blockType: BlockType.BOOLEAN,
+        text: "is [OBJECT] detected with confidence > [THRESHOLD]%",
+        func: "isObjectDetected",
+        arguments: {
+          OBJECT: {
+            type: ArgumentType.STRING,
+            menu: "modelLabels",
+            defaultValue: "ok",
+          },
+          THRESHOLD: {
+            type: ArgumentType.NUMBER,
+            defaultValue: 50,
+          },
+        },
+      },
+      {
+        opcode: "getConfidence",
+        blockType: BlockType.REPORTER,
+        text: "confidence of [OBJECT]",
+        func: "getConfidence",
+        arguments: {
+          OBJECT: {
+            type: ArgumentType.STRING,
+            menu: "modelLabels",
+            defaultValue: "ok",
+          },
+        },
+      },
+    ],
+    menus: {
+      modelLabels: "getModelLabels",
+    },
+  };
+};
+
+TeachableMachineImage.prototype.getModelLabels = function() {
+  return this.modelLabels.length > 0 ? this.modelLabels : ["ok", "stop", "unknown"];
+};
+
+TeachableMachineImage.prototype.whenObjectDetected = function(args) {
+  const confidence = this.getConfidence(args.OBJECT);
+  return confidence > (args.THRESHOLD / 100);
+};
+
+TeachableMachineImage.prototype.isObjectDetected = function(args) {
+  const confidence = this.getConfidence(args.OBJECT);
+  return confidence > (args.THRESHOLD / 100);
+};
+
+TeachableMachineImage.prototype.getConfidence = function(args) {
+  if (!this.predictions || this.predictions.length === 0) return 0;
+
+  const prediction = this.predictions.find(p => p.className === args.OBJECT);
+  const confidence = prediction ? Math.round(prediction.probability * 100) : 0;
+
+  console.log("get confidence for", args.OBJECT, "=", confidence + "%");
+  return confidence;
+};
+
+TeachableMachineImage.prototype.startDetectionLoop = function(args) {
+  this.runtime.ioDevices.video.enableVideo();
+  this.startPredictionLoop();
+};
+
+module.exports = TeachableMachineImage;

scratch-arduino-extensions/scripts/patch-gui.js

@@ -5,6 +5,7 @@ const extensions = [
   { name: "ArduinoBasics", directory: "arduino_basics" },
   { name: "ArduinoModulino", directory: "arduino_modulino" },
   { name: "ArduinoObjectDetection", directory: "arduino_object_detection" },
+  { name: "TeachableMachineImage", directory: "tmachine_image" },
 ];
 
 // base dir is the 'scratch-arduino-extensions' folder