Best-effort rendering when GPU does not support full dataset (#4424)

* show better gpu error messages
* temporarily disable some CI steps
* remove accidental error throwing
* don't expose all color layers to shader
* don't throw when not enough textures are available
* dynamically re-compile shader with least-recently used layers in case the hardware does not support accessing all textures at the same time
* warn user if more than the supported layer count is enabled
* ensure that the maximum amount of renderable layers is used from the beginning
* remove isBasicRenderingSupported flag since it's not useful anymore
* throttle shader recomputation and refactor a bit
* further clean up
* properly take segmentation layer into account
* reactivate full CI checks
* fix tests
* guard against mesa driver on firefox which does not support more than 16 samplers
* incorporate pr feedback
* linting
* remove comment
* update changelog
* Merge branch 'master' into better-gpu-error-msg-graceful-rendering

CHANGELOG.md

@@ -10,9 +10,9 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 [Commits](https://github.com/scalableminds/webknossos/compare/20.02.0...HEAD)
 
 ### Added
+- Added support for datasets with more layers than the hardware can render simultaneously. The user can disable layers temporarily to control for which layers the GPU resources should be used. [#4424](https://github.com/scalableminds/webknossos/pull/4424)
 - Added a notification when downloading nml including volume that informs that the fallback data is excluded in the download. [#4413](https://github.com/scalableminds/webknossos/pull/4413)
 
-
 ### Changed
 - Made the navbar scrollable on small screens. [#4413](https://github.com/scalableminds/webknossos/pull/4413)
 - Opening the settings sidebar when viewing a dataset or tracing defaults to the dataset settings now. [#4425](https://github.com/scalableminds/webknossos/pull/4425)

frontend/javascripts/messages.js

@@ -116,6 +116,9 @@ In order to restore the current window, a reload is necessary.`,
   "webgl.disabled": "Couldn't initialise WebGL, please make sure WebGL is enabled.",
   "webgl.context_loss":
     "Unfortunately, WebGL crashed. Please ensure that your graphics card driver is up to date to avoid such crashes. If this message keeps appearing, you can also try to lower the data rendering quality in the settings. Restarting your browser might also help.",
+  "webgl.too_many_active_layers": _.template(
+    "Your hardware cannot render all layers of this dataset simultaneously. Please ensure that not more than <%- maximumLayerCountToRender %> layers are enabled in the sidebar settings.",
+  ),
   "task.user_script_retrieval_error": "Unable to retrieve script",
   "task.new_description": "You are now tracing a new task with the following description",
   "task.no_description": "You are now tracing a new task with no description.",

frontend/javascripts/oxalis/default_state.js

@@ -97,6 +97,7 @@ const defaultState: OxalisState = {
       smallestCommonBucketCapacity:
         Constants.GPU_FACTOR_MULTIPLIER * Constants.DEFAULT_GPU_MEMORY_FACTOR,
       initializedGpuFactor: Constants.GPU_FACTOR_MULTIPLIER,
+      maximumLayerCountToRender: 32,
     },
   },
   task: null,

frontend/javascripts/oxalis/geometries/materials/plane_material_factory.js

@@ -24,6 +24,7 @@ import {
   getByteCount,
   getElementClass,
   getBoundaries,
+  getEnabledColorLayers,
 } from "oxalis/model/accessors/dataset_accessor";
 import { getRequestLogZoomStep, getZoomValue } from "oxalis/model/accessors/flycam_accessor";
 import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers";
@@ -41,6 +42,8 @@ type ShaderMaterialOptions = {
   polygonOffsetUnits?: number,
 };
 
+const RECOMPILATION_THROTTLE_TIME = 500;
+
 export type Uniforms = {
   [key: string]: {
     type: "b" | "f" | "i" | "t" | "v2" | "v3" | "v4" | "tv",
@@ -60,7 +63,7 @@ function sanitizeName(name: ?string): string {
   return `layer_${btoa(name).replace(/=+/g, "")}`;
 }
 
-function getColorLayerNames() {
+function getSanitizedColorLayerNames() {
   return getColorLayers(Store.getState().dataset).map(layer => sanitizeName(layer.name));
 }
 
@@ -82,11 +85,14 @@ class PlaneMaterialFactory {
   attributes: Object;
   shaderId: number;
   storePropertyUnsubscribers: Array<() => void> = [];
+  leastRecentlyVisibleLayers: Array<string>;
+  oldShaderCode: ?string;
 
   constructor(planeID: OrthoView, isOrthogonal: boolean, shaderId: number) {
     this.planeID = planeID;
     this.isOrthogonal = isOrthogonal;
     this.shaderId = shaderId;
+    this.leastRecentlyVisibleLayers = [];
   }
 
   setup() {
@@ -210,7 +216,7 @@ class PlaneMaterialFactory {
       };
     }
 
-    for (const name of getColorLayerNames()) {
+    for (const name of getSanitizedColorLayerNames()) {
       this.uniforms[`${name}_alpha`] = {
         type: "f",
         value: 1,
@@ -414,13 +420,27 @@ class PlaneMaterialFactory {
       ),
     );
 
+    const oldVisibilityPerLayer = {};
     this.storePropertyUnsubscribers.push(
       listenToStoreProperty(
         state => state.datasetConfiguration.layers,
         layerSettings => {
           for (const colorLayer of getColorLayers(Store.getState().dataset)) {
             const settings = layerSettings[colorLayer.name];
             if (settings != null) {
+              const isLayerEnabled = !settings.isDisabled;
+              if (
+                oldVisibilityPerLayer[colorLayer.name] != null &&
+                oldVisibilityPerLayer[colorLayer.name] !== isLayerEnabled
+              ) {
+                if (settings.isDisabled) {
+                  this.onDisableLayer(colorLayer.name);
+                } else {
+                  this.onEnableLayer(colorLayer.name);
+                }
+              }
+              oldVisibilityPerLayer[colorLayer.name] = isLayerEnabled;
+
               const name = sanitizeName(colorLayer.name);
               this.updateUniformsForLayer(settings, name, colorLayer.elementClass);
             }
@@ -545,35 +565,107 @@ class PlaneMaterialFactory {
     return this.material;
   }
 
+  recomputeFragmentShader = _.throttle(() => {
+    const newShaderCode = this.getFragmentShader();
+    // Comparing to this.material.fragmentShader does not work. The code seems
+    // to be modified by a third party.
+    if (this.oldShaderCode != null && this.oldShaderCode === newShaderCode) {
+      return;
+    }
+    this.oldShaderCode = newShaderCode;
+
+    this.material.fragmentShader = newShaderCode;
+    this.material.needsUpdate = true;
+    window.needsRerender = true;
+  }, RECOMPILATION_THROTTLE_TIME);
+
+  getLayersToRender(maximumLayerCountToRender: number) {
+    // This function determines for which layers
+    // the shader code should be compiled. If the GPU supports
+    // all layers, we can simply return all layers here.
+    // Otherwise, we prioritize layers to render by taking
+    // into account (a) which layers are activated and (b) which
+    // layers were least-recently activated (but are now disabled).
+
+    if (maximumLayerCountToRender <= 0) {
+      return [];
+    }
+
+    const colorLayerNames = getSanitizedColorLayerNames();
+    if (maximumLayerCountToRender >= colorLayerNames.length) {
+      // We can simply render all available layers.
+      return colorLayerNames;
+    }
+
+    const state = Store.getState();
+    const enabledLayerNames = getEnabledColorLayers(state.dataset, state.datasetConfiguration).map(
+      layer => layer.name,
+    );
+    const disabledLayerNames = getEnabledColorLayers(state.dataset, state.datasetConfiguration, {
+      invert: true,
+    }).map(layer => layer.name);
+
+    // In case, this.leastRecentlyVisibleLayers does not contain all disabled layers
+    // because they were already disabled on page load), append the disabled layers
+    // which are not already in that array.
+    // Note that the order of this array is important (earlier elements are more "recently used")
+    // which is why it is important how this operation is done.
+    this.leastRecentlyVisibleLayers = [
+      ...this.leastRecentlyVisibleLayers,
+      ..._.without(disabledLayerNames, ...this.leastRecentlyVisibleLayers),
+    ];
+
+    return enabledLayerNames
+      .concat(this.leastRecentlyVisibleLayers)
+      .slice(0, maximumLayerCountToRender)
+      .sort()
+      .map(sanitizeName);
+  }
+
+  onDisableLayer = layerName => {
+    this.leastRecentlyVisibleLayers = _.without(this.leastRecentlyVisibleLayers, layerName);
+    this.leastRecentlyVisibleLayers = [layerName, ...this.leastRecentlyVisibleLayers];
+
+    this.recomputeFragmentShader();
+  };
+
+  onEnableLayer = layerName => {
+    this.leastRecentlyVisibleLayers = _.without(this.leastRecentlyVisibleLayers, layerName);
+    this.recomputeFragmentShader();
+  };
+
   getFragmentShader(): string {
-    const colorLayerNames = getColorLayerNames();
-    const packingDegreeLookup = getPackingDegreeLookup();
+    const {
+      initializedGpuFactor,
+      maximumLayerCountToRender,
+    } = Store.getState().temporaryConfiguration.gpuSetup;
+
     const segmentationLayer = Model.getSegmentationLayer();
+    const colorLayerNames = this.getLayersToRender(
+      maximumLayerCountToRender - (segmentationLayer ? 1 : 0),
+    );
+    const packingDegreeLookup = getPackingDegreeLookup();
     const segmentationName = sanitizeName(segmentationLayer ? segmentationLayer.name : "");
     const { dataset } = Store.getState();
     const datasetScale = dataset.dataSource.scale;
     // Don't compile code for segmentation in arbitrary mode
     const hasSegmentation = this.isOrthogonal && segmentationLayer != null;
 
-    const lookupTextureWidth = getLookupBufferSize(
-      Store.getState().temporaryConfiguration.gpuSetup.initializedGpuFactor,
-    );
+    const lookupTextureWidth = getLookupBufferSize(initializedGpuFactor);
 
-    const code = getMainFragmentShader({
+    return getMainFragmentShader({
       colorLayerNames,
       packingDegreeLookup,
       hasSegmentation,
       segmentationName,
       isMappingSupported: Model.isMappingSupported,
       // Todo: this is not computed per layer. See #4018
-      dataTextureCountPerLayer: Model.maximumDataTextureCountForLayer,
+      dataTextureCountPerLayer: Model.maximumTextureCountForLayer,
       resolutions: getResolutions(dataset),
       datasetScale,
       isOrthogonal: this.isOrthogonal,
       lookupTextureWidth,
     });
-
-    return code;
   }
 
   getVertexShader(): string {

frontend/javascripts/oxalis/model.js

@@ -27,7 +27,7 @@ export class OxalisModel {
     [key: string]: DataLayer,
   };
   isMappingSupported: boolean = true;
-  maximumDataTextureCountForLayer: number;
+  maximumTextureCountForLayer: number;
 
   async fetch(
     annotationType: AnnotationType,
@@ -47,12 +47,12 @@ export class OxalisModel {
         dataLayers,
         connectionInfo,
         isMappingSupported,
-        maximumDataTextureCountForLayer,
+        maximumTextureCountForLayer,
       } = initializationInformation;
       this.dataLayers = dataLayers;
       this.connectionInfo = connectionInfo;
       this.isMappingSupported = isMappingSupported;
-      this.maximumDataTextureCountForLayer = maximumDataTextureCountForLayer;
+      this.maximumTextureCountForLayer = maximumTextureCountForLayer;
     }
   }
 

frontend/javascripts/oxalis/model/accessors/dataset_accessor.js

@@ -315,6 +315,30 @@ export function getColorLayers(dataset: APIDataset): Array<DataLayerType> {
   return dataset.dataSource.dataLayers.filter(dataLayer => isColorLayer(dataset, dataLayer.name));
 }
 
+export function getEnabledColorLayers(
+  dataset: APIDataset,
+  datasetConfiguration: DatasetConfiguration,
+  options: { invert?: boolean } = {},
+): Array<DataLayerType> {
+  const colorLayers = getColorLayers(dataset);
+  const layerSettings = datasetConfiguration.layers;
+
+  return colorLayers.filter(layer => {
+    const settings = layerSettings[layer.name];
+    if (settings == null) {
+      return false;
+    }
+    return settings.isDisabled === options.invert;
+  });
+}
+
+export function isSegmentationLayerEnabled(
+  dataset: APIDataset,
+  datasetConfiguration: DatasetConfiguration,
+): boolean {
+  return !datasetConfiguration.isSegmentationDisabled;
+}
+
 export function getThumbnailURL(dataset: APIDataset): string {
   const datasetName = dataset.name;
   const organizationName = dataset.owningOrganization;

frontend/javascripts/oxalis/model/actions/settings_actions.js

@@ -50,6 +50,7 @@ type InitializeGpuSetupAction = {
   type: "INITIALIZE_GPU_SETUP",
   bucketCapacity: number,
   gpuFactor: number,
+  maximumLayerCountToRender: number,
 };
 type SetMappingEnabledAction = { type: "SET_MAPPING_ENABLED", isMappingEnabled: boolean };
 type SetMappingAction = {
@@ -174,8 +175,10 @@ export const setMappingAction = (
 export const initializeGpuSetupAction = (
   bucketCapacity: number,
   gpuFactor: number,
+  maximumLayerCountToRender: number,
 ): InitializeGpuSetupAction => ({
   type: "INITIALIZE_GPU_SETUP",
   bucketCapacity,
   gpuFactor,
+  maximumLayerCountToRender,
 });