[pull] ndmaster from google:master (#19)

* feat(zarr): support dimension_separator in .zarray file This implements the spec change described here: zarr-developers/zarr-python#715 This also adds a query string parameter by the same name. Fixes google#241. * feat(multiscale mesh): add additional debugging checks * fix(multiscale mesh): turn off multiscale mesh fragment debugging by default This was mistakenly left on by default. Co-authored-by: Jeremy Maitin-Shepard <jbms@google.com>
jhuapl-boss · Apr 16, 2021 · 61f5f77 · 61f5f77
1 parent b1ff70b
commit 61f5f77
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 8 deletions.
diff --git a/src/neuroglancer/datasource/precomputed/backend.ts b/src/neuroglancer/datasource/precomputed/backend.ts
@@ -43,9 +43,12 @@ import {stableStringify} from 'neuroglancer/util/json';
 import {getObjectId} from 'neuroglancer/util/object_id';
 import {cancellableFetchSpecialOk, SpecialProtocolCredentials, SpecialProtocolCredentialsProvider} from 'neuroglancer/util/special_protocol_request';
 import {Uint64} from 'neuroglancer/util/uint64';
-import {encodeZIndexCompressed} from 'neuroglancer/util/zorder';
+import {encodeZIndexCompressed, zorder3LessThan} from 'neuroglancer/util/zorder';
 import {registerSharedObject} from 'neuroglancer/worker_rpc';
 
+// Set to true to validate the multiscale index.
+const DEBUG_MULTISCALE_INDEX = false;
+
 const shardingHashFunctions: Map<ShardingHashFunction, (out: Uint64) => void> = new Map([
   [
     ShardingHashFunction.MURMURHASH3_X86_128,
@@ -387,10 +390,30 @@ function decodeMultiscaleManifestChunk(
   const clipLowerBound =
       vec3.fromValues(Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY);
   let numLods = Math.max(1, storedLodScales.length);
+  // Compute `clipLowerBound` and `clipUpperBound` and `numLods`.  Note that `numLods` is >=
+  // `storedLodScales.length`; it may contain additional levels since at the highest level the
+  // octree must be a single node.
   {
     let fragmentBase = 0;
     for (let lodIndex = 0; lodIndex < numStoredLods; ++lodIndex) {
       const numFragments = numFragmentsPerLod[lodIndex];
+      if (DEBUG_MULTISCALE_INDEX) {
+        for (let i = 1; i < numFragments; ++i) {
+          let x0 = fragmentInfo[fragmentBase + numFragments * 0 + (i - 1)];
+          let y0 = fragmentInfo[fragmentBase + numFragments * 1 + (i - 1)];
+          let z0 = fragmentInfo[fragmentBase + numFragments * 2 + (i - 1)];
+          let x1 = fragmentInfo[fragmentBase + numFragments * 0 + i];
+          let y1 = fragmentInfo[fragmentBase + numFragments * 1 + i];
+          let z1 = fragmentInfo[fragmentBase + numFragments * 2 + i];
+          if (!zorder3LessThan(x0, y0, z0, x1, y1, z1)) {
+            console.log(
+                `Fragment index violates zorder constraint: ` +
+                `lod=${lodIndex}, ` +
+                `chunk ${i - 1} = [${x0},${y0},${z0}], ` +
+                `chunk ${i} = [${x1},${y1},${z1}]`);
+          }
+        }
+      }
       for (let i = 0; i < 3; ++i) {
         let upperBoundValue = Number.NEGATIVE_INFINITY;
         let lowerBoundValue = Number.POSITIVE_INFINITY;
@@ -416,6 +439,8 @@ function decodeMultiscaleManifestChunk(
     }
   }
 
+  // Compute upper bound on number of nodes that will be in the octree, so that we can allocate a
+  // sufficiently large buffer without having to worry about resizing.
   let maxFragments = 0;
   {
     let prevNumFragments = 0;

diff --git a/src/neuroglancer/datasource/zarr/README.md b/src/neuroglancer/datasource/zarr/README.md
@@ -7,6 +7,10 @@ format arrays, using the following data source URL syntax:
 `zarr://FILE_URL`, where `FILE_URL` is a URL to the directory containing the `.zarray` metadata file
 using any [supported file protocol](../file_protocols.md).
 
+If the zarr array uses `/` rather than the default of `.` as the dimension separator in chunk keys,
+you can either specify the separator as the `dimension_separator` member in the `.zarray` metadata
+file (preferred) or use a data source URL of `zarr://FILE_URL?dimension_separator=/`.
+
 Supported compressors:
 
 - raw

diff --git a/src/neuroglancer/datasource/zarr/frontend.ts b/src/neuroglancer/datasource/zarr/frontend.ts
@@ -24,10 +24,11 @@ import {SliceViewSingleResolutionSource} from 'neuroglancer/sliceview/frontend';
 import {DataType, makeDefaultVolumeChunkSpecifications, VolumeSourceOptions, VolumeType} from 'neuroglancer/sliceview/volume/base';
 import {MultiscaleVolumeChunkSource as GenericMultiscaleVolumeChunkSource, VolumeChunkSource} from 'neuroglancer/sliceview/volume/frontend';
 import {transposeNestedArrays} from 'neuroglancer/util/array';
+import {applyCompletionOffset, completeQueryStringParametersFromTable} from 'neuroglancer/util/completion';
 import {Borrowed} from 'neuroglancer/util/disposable';
 import {completeHttpPath} from 'neuroglancer/util/http_path_completion';
 import {isNotFoundError, responseJson} from 'neuroglancer/util/http_request';
-import {parseArray, parseFixedLengthArray, verifyObject, verifyObjectProperty, verifyOptionalObjectProperty, verifyString} from 'neuroglancer/util/json';
+import {parseArray, parseFixedLengthArray, parseQueryStringParameters, verifyObject, verifyObjectProperty, verifyOptionalObjectProperty, verifyString} from 'neuroglancer/util/json';
 import {createIdentity} from 'neuroglancer/util/matrix';
 import {parseNumpyDtype} from 'neuroglancer/util/numpy_dtype';
 import {getObjectId} from 'neuroglancer/util/object_id';
@@ -43,6 +44,16 @@ interface ZarrMetadata {
   rank: number;
   shape: number[];
   chunks: number[];
+  dimensionSeparator: ZarrSeparator|undefined;
+}
+
+function parseDimensionSeparator(obj: unknown): ZarrSeparator|undefined {
+  return verifyOptionalObjectProperty(obj, 'dimension_separator', value => {
+    if (value !== '.' && value !== '/') {
+      throw new Error(`Expected "." or "/", but received: ${JSON.stringify(value)}`);
+    }
+    return value;
+  });
 }
 
 function parseZarrMetadata(obj: unknown): ZarrMetadata {
@@ -75,6 +86,7 @@ function parseZarrMetadata(obj: unknown): ZarrMetadata {
       }
       return order;
     });
+    const dimensionSeparator = parseDimensionSeparator(obj);
     const numpyDtype =
         verifyObjectProperty(obj, 'dtype', dtype => parseNumpyDtype(verifyString(dtype)));
     const compressor = verifyObjectProperty(obj, 'compressor', compressor => {
@@ -99,6 +111,7 @@ function parseZarrMetadata(obj: unknown): ZarrMetadata {
       order,
       dataType: numpyDtype.dataType,
       encoding: {compressor, endianness: numpyDtype.endianness},
+      dimensionSeparator,
     };
   } catch (e) {
     throw new Error(`Error parsing zarr metadata: ${e.message}`);
@@ -213,26 +226,47 @@ function getMetadata(
         return parseZarrMetadata(json);
       });
 }
+const supportedQueryParameters = [
+  {
+    key: {value: 'dimension_separator', description: 'Dimension separator in chunk keys'},
+    values: [
+      {value: '.', description: '(default)'},
+      {value: '/', description: ''},
+    ]
+  },
+];
 
 export class ZarrDataSource extends DataSourceProvider {
   get description() {
     return 'Zarr data source';
   }
   get(options: GetDataSourceOptions): Promise<DataSource> {
-    let {providerUrl} = options;
+    // Pattern is infallible.
+    let [, providerUrl, query] = options.providerUrl.match(/([^?]*)(?:\?(.*))?$/)!;
+    const parameters = parseQueryStringParameters(query || '');
+    verifyObject(parameters);
+    const dimensionSeparator = parseDimensionSeparator(parameters);
     if (providerUrl.endsWith('/')) {
       providerUrl = providerUrl.substring(0, providerUrl.length - 1);
     }
     return options.chunkManager.memoize.getUncounted(
-        {'type': 'zarr:MultiscaleVolumeChunkSource', providerUrl}, async () => {
+        {'type': 'zarr:MultiscaleVolumeChunkSource', providerUrl, dimensionSeparator}, async () => {
           const {url, credentialsProvider} =
               parseSpecialUrl(providerUrl, options.credentialsManager);
           const [metadata, attrs] = await Promise.all([
             getMetadata(options.chunkManager, credentialsProvider, url),
             getAttributes(options.chunkManager, credentialsProvider, url)
           ]);
+          if (metadata.dimensionSeparator !== undefined && dimensionSeparator !== undefined &&
+              metadata.dimensionSeparator !== dimensionSeparator) {
+            throw new Error(
+                `Explicitly specified dimension separator ` +
+                `${JSON.stringify(dimensionSeparator)} does not match value ` +
+                `in .zarray ${JSON.stringify(metadata.dimensionSeparator)}`);
+          }
           const volume = new MultiscaleVolumeChunkSource(
-              options.chunkManager, credentialsProvider, url, '.', metadata, attrs);
+              options.chunkManager, credentialsProvider, url,
+              dimensionSeparator || metadata.dimensionSeparator || '.', metadata, attrs);
           return {
             modelTransform: makeIdentityTransform(volume.modelSpace),
             subsources: [
@@ -256,8 +290,15 @@ export class ZarrDataSource extends DataSourceProvider {
         })
   }
 
-  completeUrl(options: CompleteUrlOptions) {
-    return completeHttpPath(
+  async completeUrl(options: CompleteUrlOptions) {
+    // Pattern is infallible.
+    let [, , query] = options.providerUrl.match(/([^?]*)(?:\?(.*))?$/)!;
+    if (query !== undefined) {
+      return applyCompletionOffset(
+          options.providerUrl.length - query.length,
+          await completeQueryStringParametersFromTable(query, supportedQueryParameters));
+    }
+    return await completeHttpPath(
         options.credentialsManager, options.providerUrl, options.cancellationToken);
   }
 }
diff --git a/src/neuroglancer/mesh/frontend.ts b/src/neuroglancer/mesh/frontend.ts
@@ -18,7 +18,7 @@ import {ChunkState} from 'neuroglancer/chunk_manager/base';
 import {Chunk, ChunkManager, ChunkSource} from 'neuroglancer/chunk_manager/frontend';
 import {VisibleLayerInfo} from 'neuroglancer/layer';
 import {EncodedMeshData, FRAGMENT_SOURCE_RPC_ID, MESH_LAYER_RPC_ID, MULTISCALE_FRAGMENT_SOURCE_RPC_ID, MULTISCALE_MESH_LAYER_RPC_ID, MultiscaleFragmentFormat, VertexPositionFormat} from 'neuroglancer/mesh/base';
-import {getMultiscaleChunksToDraw, getMultiscaleFragmentKey, MultiscaleMeshManifest} from 'neuroglancer/mesh/multiscale';
+import {getMultiscaleChunksToDraw, getMultiscaleFragmentKey, MultiscaleMeshManifest, validateOctree} from 'neuroglancer/mesh/multiscale';
 import {PerspectivePanel} from 'neuroglancer/perspective_view/panel';
 import {PerspectiveViewReadyRenderContext, PerspectiveViewRenderContext, PerspectiveViewRenderLayer} from 'neuroglancer/perspective_view/render_layer';
 import {ThreeDimensionalRenderLayerAttachmentState, update3dRenderLayerAttachment} from 'neuroglancer/renderlayer';
@@ -596,6 +596,9 @@ export class MultiscaleMeshLayer extends
       ++presentManifestChunks;
       const {manifest} = manifestChunk;
       const {octree, chunkShape, chunkGridSpatialOrigin, vertexOffsets} = manifest;
+      if (DEBUG_MULTISCALE_FRAGMENTS) {
+        validateOctree(octree);
+      }
       if (renderContext.emitColor) {
         meshShaderManager.setColor(gl, shader, color!);
       }

diff --git a/src/neuroglancer/mesh/multiscale.ts b/src/neuroglancer/mesh/multiscale.ts
@@ -273,6 +273,9 @@ export function validateOctree(octree: Uint32Array) {
   }
   if (numNodes === 0) return;
   exploreNode(numNodes - 1);
+  if (seenNodes.size !== numNodes) {
+    throw new Error('Orphan nodes in octree');
+  }
 }
 
 export function getMultiscaleFragmentKey(objectKey: string, lod: number, chunkIndex: number) {