Fix GPU rendering bugs, cell 0 bugs, and bug in saving raw array (#384)

* Begin work on fixing gpu bug adding numValues

* Fix bugs: GPU, cell 0, raw uint8array

backend/deepcell_label/export.py

@@ -68,7 +68,6 @@ def load_raw(self):
         with zipfile.ZipFile(self.labels_zip) as zf:
             with zf.open('raw.dat') as f:
                 raw = np.frombuffer(f.read(), self.dtype)
-                print(raw.dtype)
                 self.raw = np.reshape(
                     raw, (self.num_channels, self.duration, self.height, self.width)
                 )

frontend/src/Project/Canvas/ToolCanvas/OuterOutlineCanvas.js

@@ -33,7 +33,7 @@ function OuterOutlineCanvas({ setBitmaps, cell, color }) {
 
   useEffect(() => {
     const kernel = gpu.createKernel(
-      `function (data, cells, cell, color) {
+      `function (data, cells, numValues, cell, color) {
         const x = this.thread.x;
         const y = this.constants.h - 1 - this.thread.y;
         const value = data[y][x];
@@ -53,10 +53,12 @@ function OuterOutlineCanvas({ setBitmaps, cell, color }) {
         if (y !== this.constants.h - 1) {
           east = data[y + 1][x];
         }
-        if (cells[value][cell] === 0) {
+        if (cells[value][cell] === 0 && value < numValues) {
           if (cells[north][cell] === 1 || cells[south][cell] === 1 || cells[west][cell] === 1 || cells[east][cell] === 1) {
-            const [r, g, b, a] = color;
-            this.color(r, g, b, a);
+            if (north < numValues && south < numValues && west < numValues && east < numValues) {
+              const [r, g, b, a] = color;
+              this.color(r, g, b, a);
+            }
           }
         }
       }`,
@@ -80,7 +82,8 @@ function OuterOutlineCanvas({ setBitmaps, cell, color }) {
         return rest;
       });
     } else if (labeledArray && cellMatrix) {
-      kernel(labeledArray, cellMatrix, cell, color);
+      const numValues = cellMatrix.length;
+      kernel(labeledArray, cellMatrix, numValues, cell, color);
       // Rerender the parent canvas
       createImageBitmap(kernel.canvas).then((bitmap) => {
         setBitmaps((bitmaps) => ({ ...bitmaps, tool: bitmap }));

frontend/src/Project/Canvas/ToolCanvas/OutlineCellCanvas.js

@@ -33,7 +33,7 @@ function OutlineCellCanvas({ setBitmaps, cell, color }) {
 
   useEffect(() => {
     const kernel = gpu.createKernel(
-      `function (data, cells, cell, color) {
+      `function (data, cells, numValues, cell, color) {
         const x = this.thread.x;
         const y = this.constants.h - 1 - this.thread.y;
         const value = data[y][x];
@@ -53,8 +53,9 @@ function OutlineCellCanvas({ setBitmaps, cell, color }) {
         if (y !== this.constants.h - 1) {
           east = data[y + 1][x];
         }
-        if (cells[value][cell] === 1) {
-          if (cells[north][cell] === 0 || cells[south][cell] === 0 || cells[west][cell] === 0 || cells[east][cell] === 0) {
+        if (cells[value][cell] === 1 && value < numValues) {
+          if (cells[north][cell] === 0 || cells[south][cell] === 0 || cells[west][cell] === 0 || cells[east][cell] === 0
+              || north >= numValues || south >= numValues || west >= numValues || east >= numValues) {
             const [r, g, b, a] = color;
             this.color(r, g, b, a);
           }
@@ -80,7 +81,8 @@ function OutlineCellCanvas({ setBitmaps, cell, color }) {
         return rest;
       });
     } else if (labeledArray && cellMatrix) {
-      kernel(labeledArray, cellMatrix, cell, color);
+      const numValues = cellMatrix.length;
+      kernel(labeledArray, cellMatrix, numValues, cell, color);
       // Rerender the parent canvas
       createImageBitmap(kernel.canvas).then((bitmap) => {
         setBitmaps((bitmaps) => ({ ...bitmaps, tool: bitmap }));

frontend/src/Project/EditControls/SegmentControls/ActionButtons/AutofitButton.js

@@ -1,13 +1,21 @@
 import { useSelector } from '@xstate/react';
 import React, { useCallback } from 'react';
-import { useEditSegment } from '../../../ProjectContext';
+import { useEditSegment, useSelect } from '../../../ProjectContext';
 import ActionButton from './ActionButton';
 
 function AutofitButton(props) {
   const segment = useEditSegment();
   const grayscale = useSelector(segment, (state) => state.matches('display.grayscale'));
 
-  const onClick = useCallback(() => segment.send('AUTOFIT'), [segment]);
+  const select = useSelect();
+  const cell = useSelector(select, (state) => state.context.selected);
+
+  const onClick = useCallback(() => {
+    // Do not fit the background "cell 0"
+    if (cell !== 0) {
+      segment.send('AUTOFIT');
+    }
+  }, [segment, select, cell]);
 
   const tooltipText = (
     <span>

frontend/src/Project/EditControls/SegmentControls/ToolButtons/ThresholdButton.js

@@ -1,17 +1,22 @@
 import { useSelector } from '@xstate/react';
 import React, { useCallback } from 'react';
-import { useEditSegment } from '../../../ProjectContext';
+import { useEditSegment, useSelect } from '../../../ProjectContext';
 import ToolButton from './ToolButton';
 
 function ThresholdButton(props) {
   const segment = useEditSegment();
   const tool = useSelector(segment, (state) => state.context.tool);
   const grayscale = useSelector(segment, (state) => state.matches('display.grayscale'));
 
-  const onClick = useCallback(
-    () => segment.send({ type: 'SET_TOOL', tool: 'threshold' }),
-    [segment]
-  );
+  const select = useSelect();
+  const cell = useSelector(select, (state) => state.context.selected);
+
+  const onClick = useCallback(() => {
+    segment.send({ type: 'SET_TOOL', tool: 'threshold' });
+    if (cell === 0) {
+      select.send('SELECT_NEW');
+    }
+  }, [segment, select, cell]);
 
   const tooltipText = grayscale ? (
     <span>

frontend/src/Project/service/edit/segment/editSegmentMachine.js

@@ -90,7 +90,7 @@ const createEditSegmentMachine = (context) =>
         },
       },
       on: {
-        EXIT: { actions: 'forwardToTool' },
+        EXIT: { actions: ['forwardToTool', send({ type: 'SET_TOOL', tool: 'select' })] },
         // from canvas event bus (forwarded from parent)
         mousedown: { actions: 'forwardToTool' },
         mouseup: { actions: 'forwardToTool' },

frontend/src/Project/service/loadMachine.ts

@@ -122,7 +122,7 @@ async function getRawRasters(source: TiffPixelSource) {
     }
     channels.push(frames);
   }
-  const reshaped = reshapeRaw(channels, min, max) as Uint8Array[][][];
+  const reshaped = reshapeRaw(channels, min, max);
   return reshaped;
 }
 
@@ -145,17 +145,28 @@ async function getLabelRasters(source: TiffPixelSource) {
 }
 
 function reshapeRaw(channels: TypedArray[][][], min: number, max: number) {
+  const size_c = channels.length;
+  const size_z = channels[0].length;
+  const size_y = channels[0][0].length;
+  const size_x = channels[0][0][0].length;
+  const reshaped = [];
   // Normalize each pixel to 0-255 for rendering
-  for (let c = 0; c < channels.length; c++) {
-    for (let z = 0; z < channels[c].length; z++) {
-      for (let y = 0; y < channels[c][z].length; y++) {
-        for (let x = 0; x < channels[c][z][y].length; x++) {
-          channels[c][z][y][x] = Math.round(((channels[c][z][y][x] - min) / (max - min)) * 255);
+  for (let c = 0; c < size_c; c++) {
+    const frames = [];
+    for (let z = 0; z < size_z; z++) {
+      const frame = [];
+      for (let y = 0; y < size_y; y++) {
+        const row = new Uint8Array(size_x);
+        for (let x = 0; x < size_x; x++) {
+          row[x] = Math.round(((channels[c][z][y][x] - min) / (max - min)) * 255);
         }
+        frame.push(row);
       }
+      frames.push(frame);
     }
+    reshaped.push(frames);
   }
-  return channels;
+  return reshaped;
 }
 
 type TypedArray =