run.md

layout	title	subtitle
page	Run	Make an inference on your browser

Thanks to TensorFlowJS, the network runs on your device: the image is not sent to the server

Use landscape pictures

First inference is slower (model is not cached yet)

Interface is blocked during inference

if possible, enable WebGL

<script> class Pydnet { /** * Initializes Pydnet network. */ async init(urls) { const MODEL = "{{site.baseurl}}/assets/js/pydnet.json" this.model = await tf.loadGraphModel(MODEL); this.height = 384 this.width = 640 return this } /** * Run Pydnet * @param {Image} input image * @return {TypedArray.int32} prediction of the network */ async predict(img) { const [data, resizeInputData] = tf.tidy(() => { var raw_input = tf.browser.fromPixels(img) var upsampledraw_input = tf.image.resizeBilinear(raw_input, [this.height, this.width]) var preprocessedInput = upsampledraw_input.expandDims() preprocessedInput = tf.div(preprocessedInput, 255.0) var result = this.model.predict(preprocessedInput); result = this.prepareOutput(result, img.width, img.height); upsampledraw_input = tf.cast(upsampledraw_input, 'int32') const data = result.dataSync(); const resizeInputData = upsampledraw_input.dataSync(); return [data, resizeInputData] }); await tf.nextFrame(); return [data, resizeInputData] } /** * Run Pydnet * @param {Tensor} output depth or inverse depth predicted by the network * @param {int} width of the image * @param {int} height of the image * @return {TypedArray.int32} prediction of the network */ prepareOutput(tensor, width, height) { return tf.tidy(() => { tensor = tf.relu(tensor) tensor = tf.squeeze(tensor) var min_value = tf.min(tensor) var max_value = tf.max(tensor) tensor = tf.div(tf.sub(tensor,min_value), tf.sub(max_value,min_value)) tensor = tf.mul(tensor, 255.0) tensor = tf.cast(tensor, 'int32') return tensor }); } } async function runInference(element) { var fr = new FileReader; fr.onload = function () { var img = new Image; img.onload = function () { display_result(img) }; img.src = fr.result; }; fr.readAsDataURL(element.prop('files')[0]); } async function run_inference(img) { var outputs = await model.predict(img) return outputs } async function display_input(img, element, width, height) { var canvas = document.createElement('canvas') canvas.width = width canvas.height = height const ctx = canvas.getContext('2d'); var resizedImg = TensorToImage.apply(img, width, height) const imageData = new ImageData(resizedImg, width, height); ctx.putImageData(imageData, 0, 0); dataUrl = canvas.toDataURL() element.attr("src", dataUrl) } async function display_result(img) { var results = await run_inference(img) displayTensor(results[0], $("#outputImg"), model.width, model.height) display_input(results[1], $("#inputImg"), model.width, model.height) } function displayTensor(data, output_element, width, height) { var canvas = document.createElement('canvas') canvas.width = width canvas.height = height const ctx = canvas.getContext('2d'); var buffer = MagmaColorMap.apply(data, width, height) const imageData = new ImageData(buffer, width, height) ctx.putImageData(imageData, 0, 0) dataUrl = canvas.toDataURL() output_element.attr("src", dataUrl) } /** * From Tensor to Image * @param {Tensor} tensor with resized input image * @param {int} width of the image * @param {int} height of the image * @return {Uint8ClampedArray} resized tensor as Uint8ClampedArray image */ class TensorToImage{ static apply(buffer, width, height){ var image = new Uint8ClampedArray(width * height * 4) var i = 0 var index = 0 for (var y = 0; y < height; y++) { for (var x = 0; x < width; x++) { image[i] = buffer[index] image[i + 1] = buffer[index+1] image[i + 2] = buffer[index+2] image[i + 3] = 255.0 i += 4 index+= 3 } } return image } } class MagmaColorMap { static apply(buffer, width, height) { // adapted from https://observablehq.com/@mbostock/convert-from-tensor-to-image // NOTE: not the best way to handle the colormap, isn't it? var colorMap = new Uint8ClampedArray(width * height * 4) var magma_values = [[0, 0, 3], [0, 0, 4], [0, 0, 6], [1, 0, 7], [1, 1, 9], [1, 1, 11], [2, 2, 13], [2, 2, 15], [3, 3, 17], [4, 3, 19], [4, 4, 21], [5, 4, 23], [6, 5, 25], [7, 5, 27], [8, 6, 29], [9, 7, 31], [10, 7, 34], [11, 8, 36], [12, 9, 38], [13, 10, 40], [14, 10, 42], [15, 11, 44], [16, 12, 47], [17, 12, 49], [18, 13, 51], [20, 13, 53], [21, 14, 56], [22, 14, 58], [23, 15, 60], [24, 15, 63], [26, 16, 65], [27, 16, 68], [28, 16, 70], [30, 16, 73], [31, 17, 75], [32, 17, 77], [34, 17, 80], [35, 17, 82], [37, 17, 85], [38, 17, 87], [40, 17, 89], [42, 17, 92], [43, 17, 94], [45, 16, 96], [47, 16, 98], [48, 16, 101], [50, 16, 103], [52, 16, 104], [53, 15, 106], [55, 15, 108], [57, 15, 110], [59, 15, 111], [60, 15, 113], [62, 15, 114], [64, 15, 115], [66, 15, 116], [67, 15, 117], [69, 15, 118], [71, 15, 119], [72, 16, 120], [74, 16, 121], [75, 16, 121], [77, 17, 122], [79, 17, 123], [80, 18, 123], [82, 18, 124], [83, 19, 124], [85, 19, 125], [87, 20, 125], [88, 21, 126], [90, 21, 126], [91, 22, 126], [93, 23, 126], [94, 23, 127], [96, 24, 127], [97, 24, 127], [99, 25, 127], [101, 26, 128], [102, 26, 128], [104, 27, 128], [105, 28, 128], [107, 28, 128], [108, 29, 128], [110, 30, 129], [111, 30, 129], [113, 31, 129], [115, 31, 129], [116, 32, 129], [118, 33, 129], [119, 33, 129], [121, 34, 129], [122, 34, 129], [124, 35, 129], [126, 36, 129], [127, 36, 129], [129, 37, 129], [130, 37, 129], [132, 38, 129], [133, 38, 129], [135, 39, 129], [137, 40, 129], [138, 40, 129], [140, 41, 128], [141, 41, 128], [143, 42, 128], [145, 42, 128], [146, 43, 128], [148, 43, 128], [149, 44, 128], [151, 44, 127], [153, 45, 127], [154, 45, 127], [156, 46, 127], [158, 46, 126], [159, 47, 126], [161, 47, 126], [163, 48, 126], [164, 48, 125], [166, 49, 125], [167, 49, 125], [169, 50, 124], [171, 51, 124], [172, 51, 123], [174, 52, 123], [176, 52, 123], [177, 53, 122], [179, 53, 122], [181, 54, 121], [182, 54, 121], [184, 55, 120], [185, 55, 120], [187, 56, 119], [189, 57, 119], [190, 57, 118], [192, 58, 117], [194, 58, 117], [195, 59, 116], [197, 60, 116], [198, 60, 115], [200, 61, 114], [202, 62, 114], [203, 62, 113], [205, 63, 112], [206, 64, 112], [208, 65, 111], [209, 66, 110], [211, 66, 109], [212, 67, 109], [214, 68, 108], [215, 69, 107], [217, 70, 106], [218, 71, 105], [220, 72, 105], [221, 73, 104], [222, 74, 103], [224, 75, 102], [225, 76, 102], [226, 77, 101], [228, 78, 100], [229, 80, 99], [230, 81, 98], [231, 82, 98], [232, 84, 97], [234, 85, 96], [235, 86, 96], [236, 88, 95], [237, 89, 95], [238, 91, 94], [238, 93, 93], [239, 94, 93], [240, 96, 93], [241, 97, 92], [242, 99, 92], [243, 101, 92], [243, 103, 91], [244, 104, 91], [245, 106, 91], [245, 108, 91], [246, 110, 91], [246, 112, 91], [247, 113, 91], [247, 115, 92], [248, 117, 92], [248, 119, 92], [249, 121, 92], [249, 123, 93], [249, 125, 93], [250, 127, 94], [250, 128, 94], [250, 130, 95], [251, 132, 96], [251, 134, 96], [251, 136, 97], [251, 138, 98], [252, 140, 99], [252, 142, 99], [252, 144, 100], [252, 146, 101], [252, 147, 102], [253, 149, 103], [253, 151, 104], [253, 153, 105], [253, 155, 106], [253, 157, 107], [253, 159, 108], [253, 161, 110], [253, 162, 111], [253, 164, 112], [254, 166, 113], [254, 168, 115], [254, 170, 116], [254, 172, 117], [254, 174, 118], [254, 175, 120], [254, 177, 121], [254, 179, 123], [254, 181, 124], [254, 183, 125], [254, 185, 127], [254, 187, 128], [254, 188, 130], [254, 190, 131], [254, 192, 133], [254, 194, 134], [254, 196, 136], [254, 198, 137], [254, 199, 139], [254, 201, 141], [254, 203, 142], [253, 205, 144], [253, 207, 146], [253, 209, 147], [253, 210, 149], [253, 212, 151], [253, 214, 152], [253, 216, 154], [253, 218, 156], [253, 220, 157], [253, 221, 159], [253, 223, 161], [253, 225, 163], [252, 227, 165], [252, 229, 166], [252, 230, 168], [252, 232, 170], [252, 234, 172], [252, 236, 174], [252, 238, 176], [252, 240, 177], [252, 241, 179], [252, 243, 181], [252, 245, 183], [251, 247, 185], [251, 249, 187], [251, 250, 189], [251, 252, 191]] var i = 0 for (var y = 0; y < height; y++) { for (var x = 0; x < width; x++) { var index = y * width + x var depth = buffer[index] var rgb = magma_values[depth] colorMap[i] = rgb[0] colorMap[i + 1] = rgb[1] colorMap[i + 2] = rgb[2] colorMap[i + 3] = 255.0 i += 4 } } return colorMap } } async function setupPydnet() { model = await new Pydnet().init() } setupPydnet() </script>