From 8cb6093481f9fcc31378960b56ae5b8b932f834b Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 14:06:49 -0800 Subject: [PATCH 1/8] Don't load the pixel and cell FOVs in batches --- templates/2_Cluster_Pixels.ipynb | 29 +++++++++++++++-------------- templates/3_Cluster_Cells.ipynb | 29 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 3cce50fe5..5d3ac7e8e 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -793,21 +793,22 @@ "source": [ "subset_pixel_fovs = pixel_fovs\n", "\n", - "pixel_cluster_masks = load_utils.load_imgs_from_dir(\n", - " data_dir = os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n", - " files=[f + \"_pixel_mask.tiff\" for f in subset_pixel_fovs],\n", - " trim_suffix=\"_pixel_mask\",\n", - " match_substring=\"_pixel_mask\",\n", - " xr_dim_name=\"pixel_mask\",\n", - " xr_channel_names=None,\n", - ")\n", + "for pixel_fov in subset_pixel_fovs:\n", + " pixel_cluster_mask = load_utils.load_imgs_from_dir(\n", + " data_dir=os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n", + " files=[pixel_fov + \"_pixel_mask.tiff\"],\n", + " trim_suffix=\"_pixel_mask\",\n", + " match_substring=\"_pixel_mask\",\n", + " xr_dim_name=\"pixel_mask\",\n", + " xr_channel_names=None,\n", + " )\n", "\n", - "plot_utils.plot_pixel_cell_cluster_overlay(\n", - " pixel_cluster_masks,\n", - " pixel_fovs,\n", - " os.path.join(base_dir, pixel_meta_cluster_remap_name),\n", - " metacluster_colors=raw_cmap\n", - ")" + " plot_utils.plot_pixel_cell_cluster_overlay(\n", + " pixel_cluster_masks,\n", + " [pixel_fov],\n", + " os.path.join(base_dir, pixel_meta_cluster_remap_name),\n", + " metacluster_colors=raw_cmap\n", + " )" ] }, { diff --git a/templates/3_Cluster_Cells.ipynb b/templates/3_Cluster_Cells.ipynb index e45ce476c..70e9448e3 100644 --- a/templates/3_Cluster_Cells.ipynb +++ b/templates/3_Cluster_Cells.ipynb @@ -658,21 +658,22 @@ "source": [ "subset_cell_fovs = cell_fovs\n", "\n", - "cell_cluster_masks = load_utils.load_imgs_from_dir(\n", - " data_dir = os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", - " files=[f + \"_cell_mask.tiff\" for f in subset_cell_fovs],\n", - " trim_suffix=\"_cell_mask\",\n", - " match_substring=\"_cell_mask\",\n", - " xr_dim_name=\"cell_mask\",\n", - " xr_channel_names=None,\n", - ")\n", + "for cell_fov in subset_cell_fovs:\n", + " cell_cluster_masks = load_utils.load_imgs_from_dir(\n", + " data_dir = os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", + " files=[cell_fov + \"_cell_mask.tiff\"],\n", + " trim_suffix=\"_cell_mask\",\n", + " match_substring=\"_cell_mask\",\n", + " xr_dim_name=\"cell_mask\",\n", + " xr_channel_names=None,\n", + " )\n", "\n", - "plot_utils.plot_pixel_cell_cluster_overlay(\n", - " cell_cluster_masks,\n", - " cell_fovs,\n", - " os.path.join(base_dir, cell_meta_cluster_remap_name),\n", - " metacluster_colors=raw_cmap\n", - ")" + " plot_utils.plot_pixel_cell_cluster_overlay(\n", + " cell_cluster_masks,\n", + " [cell_fov],\n", + " os.path.join(base_dir, cell_meta_cluster_remap_name),\n", + " metacluster_colors=raw_cmap\n", + " )" ] }, { From d3faa0ba3e40f3405d6129b9980179f7cb8946d8 Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 14:10:25 -0800 Subject: [PATCH 2/8] Add patch for neighborhood mask generation --- templates/3_Cluster_Cells.ipynb | 4 ++-- ...example_neighborhood_analysis_script.ipynb | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/templates/3_Cluster_Cells.ipynb b/templates/3_Cluster_Cells.ipynb index 70e9448e3..295c414bc 100644 --- a/templates/3_Cluster_Cells.ipynb +++ b/templates/3_Cluster_Cells.ipynb @@ -659,7 +659,7 @@ "subset_cell_fovs = cell_fovs\n", "\n", "for cell_fov in subset_cell_fovs:\n", - " cell_cluster_masks = load_utils.load_imgs_from_dir(\n", + " cell_cluster_mask = load_utils.load_imgs_from_dir(\n", " data_dir = os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", " files=[cell_fov + \"_cell_mask.tiff\"],\n", " trim_suffix=\"_cell_mask\",\n", @@ -669,7 +669,7 @@ " )\n", "\n", " plot_utils.plot_pixel_cell_cluster_overlay(\n", - " cell_cluster_masks,\n", + " cell_cluster_mask,\n", " [cell_fov],\n", " os.path.join(base_dir, cell_meta_cluster_remap_name),\n", " metacluster_colors=raw_cmap\n", diff --git a/templates/example_neighborhood_analysis_script.ipynb b/templates/example_neighborhood_analysis_script.ipynb index 6eb1e6898..39914e683 100644 --- a/templates/example_neighborhood_analysis_script.ipynb +++ b/templates/example_neighborhood_analysis_script.ipynb @@ -487,16 +487,17 @@ ], "source": [ "# load the masks in and plot them\n", - "neighborhood_masks = load_utils.load_imgs_from_dir(\n", - " data_dir=overlay_out_dir,\n", - " files=[f + \"_neighborhood_mask.tiff\" for f in fovs],\n", - " trim_suffix=\"_neighborhood_mask\",\n", - " match_substring=\"_neighborhood_mask\",\n", - " xr_dim_name=\"neighborhood_mask\",\n", - " xr_channel_names=None,\n", - ")\n", + "for fov in fovs:\n", + " neighborhood_mask = load_utils.load_imgs_from_dir(\n", + " data_dir=overlay_out_dir,\n", + " files=[fov + \"_neighborhood_mask.tiff\"],\n", + " trim_suffix=\"_neighborhood_mask\",\n", + " match_substring=\"_neighborhood_mask\",\n", + " xr_dim_name=\"neighborhood_mask\",\n", + " xr_channel_names=None,\n", + " )\n", "\n", - "plot_utils.plot_neighborhood_cluster_result(neighborhood_masks, fovs, k=k, save_dir=None)" + " plot_utils.plot_neighborhood_cluster_result(neighborhood_mask, [fov], k=k, save_dir=None)" ] }, { From 22908fbbd3dec5e550a61fee2311597b9774fa74 Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 14:30:27 -0800 Subject: [PATCH 3/8] Typo in pixel cluster mask generation --- templates/2_Cluster_Pixels.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 5d3ac7e8e..0b72fec02 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -804,7 +804,7 @@ " )\n", "\n", " plot_utils.plot_pixel_cell_cluster_overlay(\n", - " pixel_cluster_masks,\n", + " pixel_cluster_mask,\n", " [pixel_fov],\n", " os.path.join(base_dir, pixel_meta_cluster_remap_name),\n", " metacluster_colors=raw_cmap\n", From 05d4765dbd218d7fbcc4b6d2506b4195bfcca896 Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 15:02:04 -0800 Subject: [PATCH 4/8] Simplify variable naming in mask generation --- templates/2_Cluster_Pixels.ipynb | 6 ++---- templates/3_Cluster_Cells.ipynb | 4 +--- templates/example_neighborhood_analysis_script.ipynb | 8 ++++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 0b72fec02..9beaf6579 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -685,7 +685,7 @@ "outputs": [], "source": [ "# select fovs to display\n", - "pixel_fovs = ['fov0', 'fov1']" + "subset_pixel_fovs = ['fov0', 'fov1']" ] }, { @@ -791,9 +791,7 @@ } ], "source": [ - "subset_pixel_fovs = pixel_fovs\n", - "\n", - "for pixel_fov in subset_pixel_fovs:\n", + "for pixel_fov in pixel_fovs:\n", " pixel_cluster_mask = load_utils.load_imgs_from_dir(\n", " data_dir=os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n", " files=[pixel_fov + \"_pixel_mask.tiff\"],\n", diff --git a/templates/3_Cluster_Cells.ipynb b/templates/3_Cluster_Cells.ipynb index 295c414bc..2a805466f 100644 --- a/templates/3_Cluster_Cells.ipynb +++ b/templates/3_Cluster_Cells.ipynb @@ -613,7 +613,7 @@ "outputs": [], "source": [ "# select fovs to display\n", - "cell_fovs = ['fov0', 'fov1']" + "subset_cell_fovs = ['fov0', 'fov1']" ] }, { @@ -656,8 +656,6 @@ }, "outputs": [], "source": [ - "subset_cell_fovs = cell_fovs\n", - "\n", "for cell_fov in subset_cell_fovs:\n", " cell_cluster_mask = load_utils.load_imgs_from_dir(\n", " data_dir = os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", diff --git a/templates/example_neighborhood_analysis_script.ipynb b/templates/example_neighborhood_analysis_script.ipynb index 39914e683..1847f2c36 100644 --- a/templates/example_neighborhood_analysis_script.ipynb +++ b/templates/example_neighborhood_analysis_script.ipynb @@ -457,11 +457,11 @@ "outputs": [], "source": [ "# select fovs to display\n", - "fovs = ['fov6']\n", + "subset_neighborhood_fovs = ['fov6']\n", "\n", "# generate and save the neighborhood masks for the fovs to display\n", "data_utils.generate_and_save_neighborhood_cluster_masks(\n", - " fovs,\n", + " subset_neighborhood_fovs,\n", " overlay_out_dir,\n", " all_data_cluster_labeled,\n", " seg_output,\n", @@ -487,7 +487,7 @@ ], "source": [ "# load the masks in and plot them\n", - "for fov in fovs:\n", + "for fov in subset_neighborhood_fovs:\n", " neighborhood_mask = load_utils.load_imgs_from_dir(\n", " data_dir=overlay_out_dir,\n", " files=[fov + \"_neighborhood_mask.tiff\"],\n", @@ -497,7 +497,7 @@ " xr_channel_names=None,\n", " )\n", "\n", - " plot_utils.plot_neighborhood_cluster_result(neighborhood_mask, [fov], k=k, save_dir=None)" + " plot_utils.plot_neighborhood_cluster_result(neighborhood_mask, [fov], k=k, save_dir=None)n" ] }, { From 045ef7a932f078cf6ce9b50bdf0155d3c73d61cf Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 15:13:17 -0800 Subject: [PATCH 5/8] Make sure we're only using the subset of FOVs to generate --- templates/2_Cluster_Pixels.ipynb | 5 ++--- templates/3_Cluster_Cells.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 9beaf6579..537b510a2 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -709,9 +709,8 @@ " )\n", "\n", "# generate and save the pixel cluster masks for each fov in pixel_fovs\n", - "\n", "data_utils.generate_and_save_pixel_cluster_masks(\n", - " fovs=pixel_fovs,\n", + " fovs=subset_pixel_fovs,\n", " base_dir=base_dir,\n", " save_dir=os.path.join(base_dir, pixel_output_dir),\n", " tiff_dir=tiff_dir,\n", @@ -879,7 +878,7 @@ "outputs": [], "source": [ "plot_utils.create_mantis_dir(\n", - " fovs=pixel_fovs,\n", + " fovs=subset_pixel_fovs,\n", " mantis_project_path=os.path.join(base_dir, pixel_output_dir, \"mantis\"),\n", " img_data_path=tiff_dir,\n", " mask_output_dir=os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n", diff --git a/templates/3_Cluster_Cells.ipynb b/templates/3_Cluster_Cells.ipynb index 2a805466f..967a33966 100644 --- a/templates/3_Cluster_Cells.ipynb +++ b/templates/3_Cluster_Cells.ipynb @@ -628,7 +628,7 @@ "source": [ "# generate and save the cell cluster masks for each fov in cell_fovs\n", "data_utils.generate_and_save_cell_cluster_masks(\n", - " fovs=cell_fovs,\n", + " fovs=subset_cell_fovs,\n", " base_dir=base_dir,\n", " save_dir=os.path.join(base_dir, \"pixie\", cell_output_dir),\n", " seg_dir=os.path.join(base_dir, segmentation_dir, \"deepcell_output\"),\n", @@ -723,7 +723,7 @@ "outputs": [], "source": [ "plot_utils.create_mantis_dir(\n", - " fovs=cell_fovs,\n", + " fovs=subset_cell_fovs,\n", " mantis_project_path=os.path.join(base_dir, \"pixie\", cell_output_dir, \"mantis\"),\n", " img_data_path=tiff_dir,\n", " mask_output_dir=os.path.join(base_dir, \"pixie\", cell_output_dir, \"cell_masks\"),\n", From 7f751b8501c5a5abc1424d7bf6000d0bdaa5d4c3 Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 15:20:51 -0800 Subject: [PATCH 6/8] Fix another instance of pixel_fovs to subset_pixel_fovs --- templates/2_Cluster_Pixels.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 537b510a2..97e2eb578 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -790,7 +790,7 @@ } ], "source": [ - "for pixel_fov in pixel_fovs:\n", + "for pixel_fov in subset_pixel_fovs:\n", " pixel_cluster_mask = load_utils.load_imgs_from_dir(\n", " data_dir=os.path.join(base_dir, pixel_output_dir, \"pixel_masks\"),\n", " files=[pixel_fov + \"_pixel_mask.tiff\"],\n", From 8a7775ec3a731aaedc9a87ef54aeedb8824736b4 Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Wed, 9 Nov 2022 15:36:39 -0800 Subject: [PATCH 7/8] Make sure comment variable matches up with actual variable --- templates/2_Cluster_Pixels.ipynb | 2 +- templates/3_Cluster_Cells.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/2_Cluster_Pixels.ipynb b/templates/2_Cluster_Pixels.ipynb index 97e2eb578..ba9c8c5c8 100644 --- a/templates/2_Cluster_Pixels.ipynb +++ b/templates/2_Cluster_Pixels.ipynb @@ -708,7 +708,7 @@ " img_sub_folder, io_utils.list_files(os.path.join(tiff_dir, fovs[0], img_sub_folder), substrs=['.tiff'])[0]\n", " )\n", "\n", - "# generate and save the pixel cluster masks for each fov in pixel_fovs\n", + "# generate and save the pixel cluster masks for each fov in subset_pixel_fovs\n", "data_utils.generate_and_save_pixel_cluster_masks(\n", " fovs=subset_pixel_fovs,\n", " base_dir=base_dir,\n", diff --git a/templates/3_Cluster_Cells.ipynb b/templates/3_Cluster_Cells.ipynb index 967a33966..ef13363ad 100644 --- a/templates/3_Cluster_Cells.ipynb +++ b/templates/3_Cluster_Cells.ipynb @@ -626,7 +626,7 @@ }, "outputs": [], "source": [ - "# generate and save the cell cluster masks for each fov in cell_fovs\n", + "# generate and save the cell cluster masks for each fov in subset_cell_fovs\n", "data_utils.generate_and_save_cell_cluster_masks(\n", " fovs=subset_cell_fovs,\n", " base_dir=base_dir,\n", From bd5a7b42006f3247336bd4dd78600a91e211f88a Mon Sep 17 00:00:00 2001 From: alex-l-kong Date: Tue, 15 Nov 2022 18:01:48 -0800 Subject: [PATCH 8/8] Remove trailing n --- templates/example_neighborhood_analysis_script.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/example_neighborhood_analysis_script.ipynb b/templates/example_neighborhood_analysis_script.ipynb index 1847f2c36..5d91879ed 100644 --- a/templates/example_neighborhood_analysis_script.ipynb +++ b/templates/example_neighborhood_analysis_script.ipynb @@ -497,7 +497,7 @@ " xr_channel_names=None,\n", " )\n", "\n", - " plot_utils.plot_neighborhood_cluster_result(neighborhood_mask, [fov], k=k, save_dir=None)n" + " plot_utils.plot_neighborhood_cluster_result(neighborhood_mask, [fov], k=k, save_dir=None)" ] }, {