Update error message passing

ark/phenotyping/cell_cluster_utils.py

@@ -497,9 +497,8 @@ def train_cell_som(fovs, channels, base_dir, pixel_data_dir, cell_table_path,
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging."
         )
 
     # read in the pixel channel averages table
@@ -602,9 +601,8 @@ def cluster_cells(base_dir, cluster_counts_norm_name='cluster_counts_norm.feathe
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging."
         )
 
     # compute the average pixel SOM/meta counts per cell SOM cluster
@@ -726,9 +724,8 @@ def cell_consensus_cluster(fovs, channels, base_dir, pixel_cluster_col, max_k=20
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging."
         )
 
     # compute the average pixel SOM/meta counts per cell meta cluster

ark/phenotyping/pixel_cluster_utils.py

@@ -895,9 +895,9 @@ def train_pixel_som(fovs, channels, base_dir,
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging. "
+            "For pixel SOM training, you will likely need to decrease the pixel subset proportion."
         )
 
 
@@ -1028,9 +1028,8 @@ def cluster_pixels(fovs, channels, base_dir, data_dir='pixel_mat_data',
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging."
         )
 
     # remove the data directory and rename the temp directory to the data directory
@@ -1153,9 +1152,8 @@ def pixel_consensus_cluster(fovs, channels, base_dir, max_k=20, cap=3,
             print(output.strip())
 
     if process.returncode != 0:
-        raise MemoryError(
-            "Process terminated: you likely have a memory-related error. Try increasing "
-            "your Docker memory limit."
+        raise OSError(
+            "Process terminated: please view error messages displayed above for debugging."
         )
 
     # remove the data directory and rename the temp directory to the data directory

ark/phenotyping/pixel_consensus_cluster.R

@@ -85,7 +85,20 @@ for (batchStart in seq(1, length(fovs), batchSize)) {
     # define the parallel cluster for this batch of fovs
     # NOTE: to prevent the occassional hanging first FOV issue, we need to log to an outfile
     # to "force" a return out of the foreach loop in this case
-    parallelCluster <- parallel::makeCluster(nCores, type="FORK", outfile='log.txt')
+    parallelStatus <- tryCatch(
+        {
+            parallelCluster <- parallel::makeCluster(nCores, type="FORK", outfile='log.txt')
+            0
+        },
+        error=function(cond) {
+            1
+        }
+    )
+
+    if (parallelStatus == 1) {
+        print(paste0("Too many cores (", nCores, ") specifed, reduce this using the ncores parameter"))
+        quit(status=1)
+    }
 
     # register parallel cluster for dopar
     doParallel::registerDoParallel(cl=parallelCluster)
@@ -115,23 +128,24 @@ for (batchStart in seq(1, length(fovs), batchSize)) {
                 # this won't be displayed to the user but is used as a helper to break out
                 # in the rare first FOV hang issue
                 print(paste('Done writing fov', fovs[i]))
-                0
+                c(0, '')
             },
             error=function(cond) {
                 # this won't be displayed to the user but is used as a helper to break out
                 # in the rare first FOV hang issue
                 print(paste('Error encountered for fov', fovs[i]))
-                1
+                c(1, cond)
             }
         )
 
-        data.frame(fov=fovs[i], status=status)
+        data.frame(fov=fovs[i], status=status[1], errCond=status[2])
     }
 
     # report any erroneous feather files
     for (i in 1:nrow(fovStatuses)) {
         if (fovStatuses[i, 'status'] == 1) {
-            print(paste("The data for FOV", fovStatuses[i, 'fov'], "has been corrupted, removing"))
+            print(paste("Processing for FOV", fovStatuses[i, 'fov'], "failed, removing from pipeline. Error message:"))
+            print(fovStatuses[i, 'errCond'])
             fovsProcessed <- fovsProcessed - 1
         }
     }

ark/phenotyping/run_pixel_som.R

@@ -58,7 +58,20 @@ for (batchStart in seq(1, length(fovs), batchSize)) {
     # define the parallel cluster for this batch of fovs
     # NOTE: to prevent the occassional hanging first FOV issue, we need to log to an outfile
     # to "force" a return out of the foreach loop in this case
-    parallelCluster <- parallel::makeCluster(nCores, type="FORK", outfile='log.txt')
+    parallelStatus <- tryCatch(
+        {
+            parallelCluster <- parallel::makeCluster(nCores, type="FORK", outfile='log.txt')
+            0
+        },
+        error=function(cond) {
+            1
+        }
+    )
+
+    if (parallelStatus == 1) {
+        print(paste0("Too many cores (", nCores, ") specified, reduce this using the ncores parameter"))
+        quit(status=1)
+    }
 
     # register parallel cluster for dopar
     doParallel::registerDoParallel(cl=parallelCluster)
@@ -97,23 +110,24 @@ for (batchStart in seq(1, length(fovs), batchSize)) {
                 # this won't be displayed to the user but is used as a helper to break out
                 # in the rare first FOV hang issue
                 print(paste('Done writing fov', fovs[i]))
-                0
+                c(0, '')
             },
             error=function(cond) {
                 # this won't be displayed to the user but is used as a helper to break out
                 # in the rare first FOV hang issue
                 print(paste('Error encountered for fov', fovs[i]))
-                1
+                c(1, cond)
             }
         )
 
-        data.frame(fov=fovs[i], status=status)
+        data.frame(fov=fovs[i], status=status[1], errCond=status[2])
     }
 
     # report any erroneous feather files
     for (i in 1:nrow(fovStatuses)) {
         if (fovStatuses[i, 'status'] == 1) {
-            print(paste("The data for FOV", fovStatuses[i, 'fov'], "has been corrupted, removing"))
+            print(paste("Processing for FOV", fovStatuses[i, 'fov'], "failed, removing from pipeline. Error message:"))
+            print(fovStatuses[i, 'errCond'])
             fovsProcessed <- fovsProcessed - 1
         }
     }