feat: DEV-4081: Implements Magic Wand tool for image segmentation

e2e/fragments/AtImageView.js

@@ -95,7 +95,7 @@ module.exports = {
   },
 
   /**
-   * Get  pixel color at point
+   * Get pixel color at point
    * @param {number} x
    * @param {number} y
    * @param {number[]} rgbArray

e2e/tests/helpers.js

@@ -444,6 +444,14 @@ const areEqualRGB = (a, b, tolerance) => {
   return true;
 };
 
+const setKonvaLayersOpacity = ([opacity]) => {
+  const stage = window.Konva.stages[0];
+
+  for (const layer of stage.getLayers()) {
+    layer.canvas._canvas.style.opacity = opacity;
+  }
+};
+
 const getKonvaPixelColorFromPoint = ([x, y]) => {
   const stage = window.Konva.stages[0];
   const colors = [];
@@ -731,6 +739,7 @@ module.exports = {
   getCanvasSize,
   getImageSize,
   getImageFrameSize,
+  setKonvaLayersOpacity,
   setZoom,
   whereIsPixel,
   countKonvaShapes,

e2e/tests/image.magic-wand.test.js

@@ -0,0 +1,199 @@
+/* global Feature, Scenario */
+
+const {
+  initLabelStudio,
+  hasKonvaPixelColorAtPoint,
+  setKonvaLayersOpacity,
+  serialize,
+  waitForImage,
+} = require('./helpers');
+const assert = require('assert');
+
+Feature('Test Image Magic Wand');
+
+const CLOUDSHADOW = {
+  color: '#1EAE3B',
+  rgbArray: [30, 174, 59],
+};
+const HAZE = {
+  color: '#68eee5',
+  rgbArray: [104, 238, 229],
+};
+const CLOUD = {
+  color: '#1b32de',
+  rgbArray: [27, 50, 222],
+};
+
+const config = `
+  <View>
+    <Labels name="labels_1" toName="image_1">
+      <Label hotkey="1" value="Snow" background="#F61E33" />
+      <Label hotkey="2" value="Cloud Shadow" background="${CLOUDSHADOW.color}" />
+      <Label hotkey="3" value="Haze" background="${HAZE.color}" />
+      <Label hotkey="4" value="Cloud" background="${CLOUD.color}" />
+      <Label hotkey="5" value="Exclude" background="#2C2C2B" />
+    </Labels>
+    <MagicWand name="magicwand_1" toName="image_1" />
+    <View style="width: 100%; margin-bottom: 20%; margin-side: 10%;">
+      <Image name="image_1" value="$image" zoomControl="true" zoom="true" crossOrigin="anonymous" />
+    </View>
+  </View>`;
+
+const annotationEmpty = {
+  id: '1000',
+  result: [],
+};
+
+// TODO: Change these URLs to heartex URLs, and ensure the heartex bucket allows CORS access
+// for these to work.
+const data = {
+  'image': [
+    'http://htx-pub.s3.amazonaws.com/samples/magicwand/magic_wand_scale_1_20200902_015806_26_2235_1B_AnalyticMS_00750_00750.jpg',
+    'http://htx-pub.s3.amazonaws.com/samples/magicwand/magic_wand_scale_2_20200902_015806_26_2235_1B_AnalyticMS_00750_00750.jpg',
+    'http://htx-pub.s3.amazonaws.com/samples/magicwand/magic_wand_scale_3_20200902_015806_26_2235_1B_AnalyticMS_00750_00750.jpg',
+    'http://htx-pub.s3.amazonaws.com/samples/magicwand/magic_wand_false_color_20200902_015806_26_2235_1B_AnalyticMS_00750_00750.jpg',
+  ],
+  'thumb': 'http://htx-pub.s3.amazonaws.com/samples/magicwand/magic_wand_thumbnail_20200902_015806_26_2235_1B_AnalyticMS_00750_00750.jpg',
+};
+
+async function magicWand(I, { msg, fromX, fromY, toX, toY }) {
+  I.usePlaywrightTo(msg, async ({ page }) => {
+    await page.mouse.move(fromX, fromY);
+    await page.mouse.down();
+    await page.mouse.move(toX, toY);
+    await page.mouse.up();
+  });
+  I.wait(1); // Ensure that the magic wand brush region is fully finished being created.
+}
+
+async function assertMagicWandPixel(I, x, y, assertValue, rgbArray, msg) {
+  const hasPixel = await I.executeScript(hasKonvaPixelColorAtPoint, [x, y, rgbArray, 1]);
+
+  assert.equal(hasPixel, assertValue, msg);
+}
+
+Scenario('Make sure the magic wand works in a variety of scenarios', async function({ I, LabelStudio, AtImageView, AtSidebar }) {
+  const params = {
+    config,
+    data,
+    annotations: [annotationEmpty],
+  };
+
+  I.amOnPage('/');
+
+  LabelStudio.setFeatureFlags({
+    'fflag_feat_front_dev_4081_magic_wand_tool': true,
+  });
+
+  I.executeScript(initLabelStudio, params);
+
+  AtImageView.waitForImage();
+  await AtImageView.lookForStage();
+  I.executeScript(waitForImage);
+
+  I.say('Making sure magic wand button is present');
+  I.seeElement('.lsf-toolbar__group button[aria-label="magicwand"]');
+
+  I.say('Making sure Eraser button is present');
+  I.seeElement('.lsf-toolbar__group button[aria-label="eraser"]');
+
+  I.say('Select magic wand & cloud class');
+  I.pressKey('W');
+  I.pressKey('4');
+
+  AtSidebar.seeRegions(0);
+
+  I.say('Magic wanding clouds with cloud class in upper left of image');
+  await magicWand(I, { msg: 'Fill in clouds upper left', fromX: 258, fromY: 214, toX: 650, toY: 650 });
+  await magicWand(I, { msg: 'Fill in clouds lower left', fromX: 337, fromY: 777, toX: 650, toY: 650 });
+
+  I.say('Ensuring repeated magic wands back to back with same class collapsed into single region');
+  AtSidebar.seeRegions(1);
+  AtSidebar.see('Cloud');
+
+  // Force all the magic wand regions to be a consistent color with no opacity to make testing
+  // magic wand pixel colors more robust.
+  I.say('Ensuring cloud magic wand pixels are correctly filled color');
+  await I.executeScript(setKonvaLayersOpacity, [1.0]);
+  assertMagicWandPixel(I, 0, 0, false, CLOUD.rgbArray,
+    'Far upper left corner should not have magic wand cloud class');
+  assertMagicWandPixel(I, 260, 50, true, CLOUD.rgbArray,
+    'Upper left should have magic wand cloud class');
+  assertMagicWandPixel(I, 300, 620, true, CLOUD.rgbArray,
+    'Lower left should have magic wand cloud class');
+  assertMagicWandPixel(I, 675, 650, false, CLOUD.rgbArray,
+    'Far lower right corner should not have magic wand cloud class');
+
+  // Make sure the region made from this is correct.
+  I.say('Ensuring magic wand brushregion was created correctly');
+  const result = await I.executeScript(serialize);
+  const entry = result[1];
+
+  assert.equal(entry['from_name'], 'labels_1');
+  assert.equal(entry['type'], 'labels');
+  const labels = entry['value']['labels'];
+
+  assert.equal(labels.length, 1);
+  assert.equal(labels[0], 'Cloud');
+  assert.equal(entry['value']['rle'].length > 0, true);
+
+  // Undo the bottom left area we just added, make sure its gone but our region list is still
+  // 1, then redo it and ensure its back and our region list is still 1 again.
+  I.say('Undoing last cloud magic wand and ensuring it worked correctly');
+  I.click('button[aria-label="Undo"]');
+  assertMagicWandPixel(I, 300, 620, false, CLOUD.rgbArray,
+    'Undone lower left should not have magic wand cloud class anymore');
+  assertMagicWandPixel(I, 260, 50, true, CLOUD.rgbArray,
+    'Upper left should still have magic wand cloud class');
+  AtSidebar.seeRegions(1);
+
+  I.say('Redoing last cloud magic wand and ensuring it worked correctly');
+  I.click('button[aria-label="Redo"]');
+  assertMagicWandPixel(I, 300, 620, true, CLOUD.rgbArray,
+    'Redone lower left should have magic wand cloud class again');
+  assertMagicWandPixel(I, 260, 50, true, CLOUD.rgbArray,
+    'Upper left should still have magic wand cloud class');
+  AtSidebar.seeRegions(1);
+
+  I.say('Unselecting last magic wand region');
+  I.pressKey('Escape');
+
+  // @todo currently gallery doesn't work well with CORS, so this is not covered by test
+  // Change to the false color view, zoom in, and magic wand with a new class.
+  // I.say('Changing to false-color view');
+  // I.click('[class*="gallery--"] img[src*="false_color"]');
+
+  I.say('Zooming in');
+  I.click('button[aria-label="zoom-in"]');
+  I.click('button[aria-label="zoom-in"]');
+  I.click('button[aria-label="zoom-in"]');
+  I.click('button[aria-label="zoom-in"]');
+  I.click('button[aria-label="zoom-in"]');
+
+  I.say('Selecting cloud shadow class');
+  I.pressKey('2');
+
+  I.say('Magic wanding cloud shadows with cloud shadow class in center of zoomed image');
+  await magicWand(I, { msg: 'Cloud shadow in middle of image', fromX: 390, fromY: 500, toX: 500, toY: 500 });
+
+  I.say('Ensuring new cloud shadow magic wand region gets added to sidebar');
+  AtSidebar.seeRegions(2);
+  AtSidebar.see('Cloud Shadow');
+
+  I.say('Ensuring cloud shadow magic wand pixels are correctly filled color');
+  await I.executeScript(setKonvaLayersOpacity, [1.0]);
+  assertMagicWandPixel(I, 0, 0, false, CLOUDSHADOW.rgbArray,
+    'Zoomed upper left corner should not have cloud shadow');
+  assertMagicWandPixel(I, 350, 360, true, CLOUDSHADOW.rgbArray,
+    'Center area should have magic wand cloud shadow class');
+
+  // Make sure if you have a region selected then change the class the region class changes.
+  I.say('Changing class of existing selected region to Haze should change it to new class');
+  I.pressKey('3');
+  AtSidebar.seeRegions(2);
+  AtSidebar.dontSee('Cloud Shadow');
+  AtSidebar.see('Haze');
+  await I.executeScript(setKonvaLayersOpacity, [1.0]);
+  assertMagicWandPixel(I, 350, 360, true, HAZE.rgbArray,
+    'Center area should have magic wand haze class');
+});

examples/image_magic_wand/START.md

@@ -0,0 +1,44 @@
+
+# Magic Wand for Image Segmentation
+
+![Magic Wand](/images/screenshots/image_magic_wand.png "Magic Wand")
+
+# Install
+
+## Linux & Ubuntu guide
+
+Install python and virtualenv 
+
+```bash
+# install python and virtualenv 
+apt install python3.6
+pip3 install virtualenv
+
+# setup python virtual environment 
+virtualenv -p python3 env3
+source env3/bin/activate
+
+# install requirements 
+cd backend
+pip install -r requirements.txt
+```
+
+## Cross Domain Image Access
+
+Note that if you are storing images that you'd like to apply the Magic Wand to cross-domain, such as on Google Storage Buckets, you will have to [enable CORS headers for the storage buckets to enable cross-domain pixel access](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) so that the Magic Wand can get the raw pixel data to threshold. By default browsers block JavaScript from accessing pixel-level image data unless the right CORS headers are set.
+
+As an example, if you wanted to configure a Google Storage Bucket with the right headers, you might do the following:
+
+```bash
+gsutil cors set gcp_cors_config.json gs://BUCKET-NAME
+```
+
+Note that in the gcp_cors_config.json example given in this directory that we have set `origin` to `*`, which means all origins can access that data, as well as set `responseHeader` to `*`, which means all HTTP response headers can be accessed. In a real scenario you probably want to think through the security ramifications of this for your own particular Label Studio setup.
+
+# Start
+
+Magic Wand for image segmentation:
+
+```bash
+fflag_feat_front_dev_4081_magic_wand_tool=1 python server.py -c config.json -l ../examples/image_magic_wand/config.xml -i ../examples/image_magic_wand/tasks.json -o output
+```