Merge branch 'master' into layer-name-length

scalableminds · May 11, 2023 · f38d962 · f38d962
2 parents a32a6cc + c7703f5
commit f38d962
Show file tree

Hide file tree

Showing 27 changed files with 402 additions and 207 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - In addition to drag and drop, the selected tree(s) in the Skeleton tab can also be moved into another group by right-clicking the target group and selecting "Move selected tree(s) here". [#7005](https://github.com/scalableminds/webknossos/pull/7005)
 - Added support for remote datasets encoded with [brotli](https://datatracker.ietf.org/doc/html/rfc7932). [#7041](https://github.com/scalableminds/webknossos/pull/7041)
 - Teams can be edited more straight-forwardly in a popup in the team edit page. [#7043](https://github.com/scalableminds/webknossos/pull/7043)
+- Annotations with Editable Mappings (a.k.a Supervoxel Proofreading) can now be merged. [#7026](https://github.com/scalableminds/webknossos/pull/7026)
 
 ### Changed
 - Loading of precomputed meshes got significantly faster (especially when using a mesh file for an oversegmentation with an applied agglomerate mapping). [#7001](https://github.com/scalableminds/webknossos/pull/7001)
@@ -33,6 +34,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Fixed a bug where users could sometimes not access their own time tracking information. [#7055](https://github.com/scalableminds/webknossos/pull/7055)
 - Fixed a bug in the wallTime calculation of the Voxelytics reports. [#7059](https://github.com/scalableminds/webknossos/pull/7059) 
 - Fixed a bug where thumbnails and raw data requests with non-bucket-aligned positions would show data at slightly wrong positions. [#7058](https://github.com/scalableminds/webknossos/pull/7058)
+- Fixed rare rendering bug for datasets with multiple layers and differing magnifications. [#7066](https://github.com/scalableminds/webknossos/pull/7066)
+- Fixed a bug where duplicating annotations with Editable Mappings could lead to a server-side endless loop. [#7026](https://github.com/scalableminds/webknossos/pull/7026)
 
 ### Removed
 

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -10,5 +10,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 - FossilDB needs to be opened with new additional column families editableMappingsInfo, editableMappingsAgglomerateToGraph, editableMappingsSegmentToAgglomerate.
 - For instances with existing editable mapping (a.k.a supervoxel proofreading) annotations: To keep those annotations alive, a python migration has to be run with access to your tracingstore’s FossilDB. It is recommended to do this during a webknossos downtime to avoid data loss. It needs python 3.8+ and the pip packages installable by `pip install grpcio-tools grpcio-health-checking`. Run it with `python tools/migrate-editable-mappings/migrate-editable-mappings.py -v -w -o localhost:7155`. Omit -o for a faster migration but no access to older versions of the editable mappings. The migration is idempotent.
 - The datastore now needs `brotli`. For Debian-based systems, this can be installed with `apt-get install libbrotli1`.
+- New FossilDB version 0.1.23 (`master__448` on Dockerhub) is required, compare [FossilDB PR](https://github.com/scalableminds/fossildb/pull/38).
 
 ### Postgres Evolutions:
diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala
@@ -365,14 +365,15 @@ Expects:
       } yield result
     }
 
-  // TODO: select versions per layer
   private def downloadExplorational(annotationId: String,
                                     typ: String,
                                     issuingUser: Option[User],
                                     skeletonVersion: Option[Long],
                                     volumeVersion: Option[Long],
                                     skipVolumeData: Boolean)(implicit ctx: DBAccessContext) = {
 
+    // Note: volumeVersion cannot currently be supplied per layer, see https://github.com/scalableminds/webknossos/issues/5925
+
     def skeletonToTemporaryFile(dataSet: DataSet,
                                 annotation: Annotation,
                                 organizationName: String): Fox[TemporaryFile] =

diff --git a/app/models/binary/DataSet.scala b/app/models/binary/DataSet.scala
@@ -671,8 +671,12 @@ class DataSetDataLayerDAO @Inject()(
 
   def findAllForDataSet(dataSetId: ObjectId): Fox[List[DataLayer]] =
     for {
-      rows <- run(DatasetLayers.filter(_._Dataset === dataSetId.id).result).map(_.toList)
-      rowsParsed <- Fox.combined(rows.map(parseRow(_, dataSetId)))
+      rows <- run(q"""SELECT _dataSet, name, category, elementClass, boundingBox, largestSegmentId, mappings,
+                          defaultViewConfiguration, adminViewConfiguration
+                      FROM webknossos.dataset_layers
+                      WHERE _dataset = $dataSetId
+                      ORDER BY name""".as[DatasetLayersRow])
+      rowsParsed <- Fox.combined(rows.toList.map(parseRow(_, dataSetId)))
     } yield rowsParsed
 
   private def insertLayerQuery(dataSetId: ObjectId, layer: DataLayer): SqlAction[Int, NoStream, Effect] =

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -270,11 +270,11 @@ services:
 
   # FossilDB
   fossildb:
-    image: scalableminds/fossildb:master__410
+    image: scalableminds/fossildb:master__448
     command:
       - fossildb
       - -c
-      - skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,editableMappings,editableMappingUpdates
+      - skeletons,skeletonUpdates,volumes,volumeData,volumeUpdates,editableMappings,editableMappingUpdates,editableMappingsInfo,editableMappingsAgglomerateToGraph,editableMappingsSegmentToAgglomerate
     user: ${USER_UID:-fossildb}:${USER_GID:-fossildb}
 
   fossildb-persisted:

diff --git a/fossildb/version b/fossildb/version
@@ -1 +1 @@
-0.1.22
+0.1.23
diff --git a/frontend/javascripts/admin/voxelytics/workflow_view.tsx b/frontend/javascripts/admin/voxelytics/workflow_view.tsx
@@ -18,6 +18,7 @@ import {
 import { useSearchParams, usePolling } from "libs/react_hooks";
 import Toast from "libs/toast";
 import { OxalisState } from "oxalis/store";
+import TabTitle from "oxalis/view/components/tab_title_component";
 import { getVoxelyticsWorkflow, isWorkflowAccessibleBySwitching } from "admin/admin_rest_api";
 import BrainSpinner, { BrainSpinnerWithError } from "components/brain_spinner";
 import TaskListView from "./task_list_view";
@@ -421,6 +422,7 @@ export default function WorkflowView() {
 
   return (
     <div className="container voxelytics-view">
+      <TabTitle title={`${collapsedReport.workflow.name} | WEBKNOSSOS`} />
       <TaskListView
         report={collapsedReport}
         tasksWithHierarchy={tasksWithHierarchy}

diff --git a/frontend/javascripts/oxalis/constants.ts b/frontend/javascripts/oxalis/constants.ts
@@ -320,6 +320,7 @@ const Constants = {
   MAXIMUM_DATE_TIMESTAMP: 8640000000000000,
   SCALEBAR_HEIGHT: 22,
   SCALEBAR_OFFSET: 10,
+  OBJECT_ID_STRING_LENGTH: 24,
 };
 export default Constants;
 

diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts b/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.ts
@@ -11,8 +11,6 @@ import {
   isLayerVisible,
   getLayerByName,
   getResolutionInfo,
-  getColorLayers,
-  getSegmentationLayers,
   invertAndTranspose,
   getTransformsForLayer,
 } from "oxalis/model/accessors/dataset_accessor";
@@ -103,8 +101,9 @@ export function getGlobalLayerIndexForLayerName(
 ): number {
   const sanitizer = optSanitizer || _.identity;
   const dataset = Store.getState().dataset;
-  const allLayers = [...getColorLayers(dataset), ...getSegmentationLayers(dataset)];
-  const layerIndex = allLayers.findIndex((layer) => sanitizer(layer.name) === layerName);
+  const layerIndex = dataset.dataSource.dataLayers.findIndex(
+    (layer) => sanitizer(layer.name) === layerName,
+  );
 
   return layerIndex;
 }

diff --git a/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx b/frontend/javascripts/oxalis/view/action-bar/merge_modal_view.tsx
@@ -1,5 +1,4 @@
-import { Alert, Modal, Button, Select, Form, Spin, Checkbox, Tooltip } from "antd";
-import { InfoCircleOutlined } from "@ant-design/icons";
+import { Alert, Modal, Button, Select, Form, Spin, Tooltip } from "antd";
 import { connect } from "react-redux";
 import React, { PureComponent } from "react";
 import type { Dispatch } from "redux";
@@ -15,6 +14,7 @@ import {
 import { location } from "libs/window";
 import InputComponent from "oxalis/view/components/input_component";
 import Request from "libs/request";
+import Constants from "oxalis/constants";
 import type { OxalisState, MutableTreeMap, TreeGroup } from "oxalis/store";
 import Store from "oxalis/store";
 import Toast from "libs/toast";
@@ -48,45 +48,6 @@ type MergeModalViewState = {
   isUploading: boolean;
   isFetchingData: boolean;
 };
-type ButtonWithCheckboxProps = {
-  checkboxContent: React.ReactElement<React.ComponentProps<any>, any>;
-  button: React.ReactElement<React.ComponentProps<any>, any>;
-  onButtonClick: (arg0: React.SyntheticEvent, arg1: boolean) => Promise<void> | void;
-};
-type ButtonWithCheckboxState = {
-  isChecked: boolean;
-};
-
-class ButtonWithCheckbox extends PureComponent<ButtonWithCheckboxProps, ButtonWithCheckboxState> {
-  state: ButtonWithCheckboxState = {
-    isChecked: true,
-  };
-
-  render() {
-    return (
-      <React.Fragment>
-        <Form.Item>
-          <Checkbox
-            onChange={(event) =>
-              this.setState({
-                isChecked: event.target.checked,
-              })
-            }
-            checked={this.state.isChecked}
-          >
-            {this.props.checkboxContent}
-          </Checkbox>
-        </Form.Item>
-        <Form.Item>
-          {React.cloneElement(this.props.button, {
-            // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'evt' implicitly has an 'any' type.
-            onClick: (evt) => this.props.onButtonClick(evt, this.state.isChecked),
-          })}
-        </Form.Item>
-      </React.Fragment>
-    );
-  }
-}
 
 class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
   state: MergeModalViewState = {
@@ -118,7 +79,7 @@ class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
     });
   }
 
-  async merge(url: string) {
+  async createMergedAnnotation(url: string) {
     await api.tracing.save();
     const annotation = await Request.receiveJSON(url, {
       method: "POST",
@@ -141,40 +102,50 @@ class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
     });
   };
 
-  handleMergeProject = async (event: React.SyntheticEvent, isLocalMerge: boolean) => {
+  handleMergeProject = async (event: React.SyntheticEvent) => {
     event.preventDefault();
     const { selectedProject } = this.state;
 
     if (selectedProject != null) {
-      if (isLocalMerge) {
-        const annotation = await getAnnotationCompoundInformation(
-          selectedProject,
-          APIAnnotationTypeEnum.CompoundProject,
-        );
-        this.mergeAnnotationIntoActiveTracing(annotation);
-      } else {
-        const url =
-          `/api/annotations/CompoundProject/${selectedProject}/merge/` +
-          `${this.props.annotationType}/${this.props.annotationId}`;
-        this.merge(url);
-      }
+      const url =
+        `/api/annotations/CompoundProject/${selectedProject}/merge/` +
+        `${this.props.annotationType}/${this.props.annotationId}`;
+      this.createMergedAnnotation(url);
     }
   };
 
-  handleMergeExplorativeAnnotation = async (event: React.SyntheticEvent, isLocalMerge: boolean) => {
+  handleImportProject = async (event: React.SyntheticEvent) => {
+    event.preventDefault();
+    const { selectedProject } = this.state;
+
+    if (selectedProject != null) {
+      const annotation = await getAnnotationCompoundInformation(
+        selectedProject,
+        APIAnnotationTypeEnum.CompoundProject,
+      );
+      this.mergeAnnotationIntoActiveTracing(annotation);
+    }
+  };
+
+  handleMergeExplorativeAnnotation = async (event: React.SyntheticEvent) => {
     event.preventDefault();
     const { selectedExplorativeAnnotation } = this.state;
 
     if (selectedExplorativeAnnotation != null) {
-      if (isLocalMerge) {
-        const annotation = await getAnnotationInformation(selectedExplorativeAnnotation);
-        this.mergeAnnotationIntoActiveTracing(annotation);
-      } else {
-        const url =
-          `/api/annotations/Explorational/${selectedExplorativeAnnotation}/merge/` +
-          `${this.props.annotationType}/${this.props.annotationId}`;
-        this.merge(url);
-      }
+      const url =
+        `/api/annotations/Explorational/${selectedExplorativeAnnotation}/merge/` +
+        `${this.props.annotationType}/${this.props.annotationId}`;
+      this.createMergedAnnotation(url);
+    }
+  };
+
+  handleImportExplorativeAnnotation = async (event: React.SyntheticEvent) => {
+    event.preventDefault();
+    const { selectedExplorativeAnnotation } = this.state;
+
+    if (selectedExplorativeAnnotation != null) {
+      const annotation = await getAnnotationInformation(selectedExplorativeAnnotation);
+      this.mergeAnnotationIntoActiveTracing(annotation);
     }
   };
 
@@ -216,25 +187,13 @@ class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
   }
 
   render() {
-    const mergeIntoActiveTracingCheckbox = (
-      <React.Fragment>
-        Merge into active annotation{" "}
-        <Tooltip title="If this option is enabled, trees and tree groups will be imported directly into the currently opened annotation. If not, a new explorative annotation will be created in your account.">
-          <InfoCircleOutlined
-            style={{
-              color: "gray",
-            }}
-          />
-        </Tooltip>
-      </React.Fragment>
-    );
     return (
       <Modal
         title="Merge"
         open={this.props.isOpen}
         onCancel={this.props.onOk}
         className="merge-modal"
-        width={800}
+        width={700}
         footer={null}
       >
         <Spin spinning={this.state.isUploading}>
@@ -267,17 +226,27 @@ class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
                 }))}
               />
             </Form.Item>
-
-            <ButtonWithCheckbox
-              checkboxContent={mergeIntoActiveTracingCheckbox}
-              button={
-                // @ts-expect-error ts-migrate(2322) FIXME: Type '"default"' is not assignable to type 'SizeTy... Remove this comment to see the full error message
-                <Button type="primary" size="default" disabled={this.state.selectedProject == null}>
+            <Form.Item>
+              <Tooltip title="Imports trees and tree groups (but no volume data) directly into the currently opened annotation.">
+                <Button
+                  disabled={this.state.selectedProject == null}
+                  onClick={this.handleImportProject}
+                >
+                  Import trees here
+                </Button>
+              </Tooltip>
+            </Form.Item>
+            <Form.Item>
+              <Tooltip title="Creates a new explorative annotation in your account with all merged contents of the current and selected annotations.">
+                <Button
+                  type="primary"
+                  disabled={this.state.selectedProject == null}
+                  onClick={this.handleMergeProject}
+                >
                   Merge
                 </Button>
-              }
-              onButtonClick={this.handleMergeProject}
-            />
+              </Tooltip>
+            </Form.Item>
           </Form>
 
           <Form layout="inline">
@@ -290,20 +259,33 @@ class _MergeModalView extends PureComponent<Props, MergeModalViewState> {
                 onChange={this.handleChangeMergeExplorativeAnnotation}
               />
             </Form.Item>
-            <ButtonWithCheckbox
-              checkboxContent={mergeIntoActiveTracingCheckbox}
-              button={
+            <Form.Item>
+              <Tooltip title="Imports trees and tree groups (but no volume data) directly into the currently opened annotation.">
+                <Button
+                  disabled={
+                    this.state.selectedExplorativeAnnotation.length !==
+                    Constants.OBJECT_ID_STRING_LENGTH
+                  }
+                  onClick={this.handleImportExplorativeAnnotation}
+                >
+                  Import trees here
+                </Button>
+              </Tooltip>
+            </Form.Item>
+            <Form.Item>
+              <Tooltip title="Creates a new explorative annotation in your account with all merged contents of the current and selected annotations.">
                 <Button
                   type="primary"
-                  // @ts-expect-error ts-migrate(2322) FIXME: Type '"default"' is not assignable to type 'SizeTy... Remove this comment to see the full error message
-                  size="default"
-                  disabled={this.state.selectedExplorativeAnnotation.length !== 24}
+                  disabled={
+                    this.state.selectedExplorativeAnnotation.length !==
+                    Constants.OBJECT_ID_STRING_LENGTH
+                  }
+                  onClick={this.handleMergeExplorativeAnnotation}
                 >
                   Merge
                 </Button>
-              }
-              onButtonClick={this.handleMergeExplorativeAnnotation}
-            />
+              </Tooltip>
+            </Form.Item>
           </Form>
         </Spin>
       </Modal>

diff --git a/util/src/main/scala/com/scalableminds/util/requestlogging/RequestLogging.scala b/util/src/main/scala/com/scalableminds/util/requestlogging/RequestLogging.scala
@@ -42,7 +42,7 @@ trait AbstractRequestLogging extends LazyLogging {
     val start = Instant.now
     for {
       result: Result <- block
-      executionTime = Instant.now - start
+      executionTime = Instant.since(start)
       _ = if (executionTime > durationThreshold) logTimeFormatted(executionTime, request, result)
     } yield result
   }

diff --git a/util/src/main/scala/com/scalableminds/util/time/Instant.scala b/util/src/main/scala/com/scalableminds/util/time/Instant.scala
@@ -67,6 +67,8 @@ object Instant extends FoxImplicits {
   def fromNanosecondsString(nanosecondsString: String): Instant =
     Instant(nanosecondsString.substring(0, nanosecondsString.length - 6).toLong)
 
+  def since(before: Instant): FiniteDuration = now - before
+
   private def fromStringSync(instantLiteral: String): Option[Instant] =
     tryo(java.time.Instant.parse(instantLiteral).toEpochMilli).toOption.map(timestamp => Instant(timestamp))
 

diff --git a/...nossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala b/...nossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala
@@ -22,7 +22,7 @@ class Application @Inject()(redisClient: DataStoreRedisStore, applicationHealthS
         before <- Fox.successful(Instant.now)
         _ <- redisClient.checkHealth
         _ <- Fox.bool2Fox(applicationHealthService.getRecentProblem().isEmpty) ?~> "Java Internal Errors detected"
-        _ = logger.info(s"Answering ok for Datastore health check, took ${Instant.now - before}")
+        _ = logger.info(s"Answering ok for Datastore health check, took ${Instant.since(before)}")
       } yield Ok("Ok")
     }
   }