20210707_RS-mTMA-20220119_mpleximage_data_extraction_pipeline_bue.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e588f7e-254c-450e-a763-64ab281af0b6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# multiplex image data extraction pipeline\n",
    "# =========================================\n",
    "\n",
    "# the latest version from this notebook`s template can be found here: \n",
    "# https://gitlab.com/bue/jinxif/-/blob/master/jupyter/mpleximage_data_extraction_pipeline.ipynb\n",
    "# wget https://gitlab.com/bue/jinxif/-/raw/master/jupyter/mpleximage_data_extraction_pipeline.ipynb\n",
    "\n",
    "# all jinxif functions have a docstring.\n",
    "# e.g. use help(segment.segment_spawn) to get detailed information about the segment_spawn function.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d3f8f4e-1011-49d1-b996-64abb63fedfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load libraries\n",
    "# from jinxif import _version\n",
    "# from jinxif import afsub\n",
    "# from jinxif import basic\n",
    "# from jinxif import config\n",
    "# from jinxif import feat\n",
    "# from jinxif import imgmeta\n",
    "# from jinxif import ometiff\n",
    "# from jinxif import regist\n",
    "# from jinxif import sane\n",
    "# from jinxif import segment\n",
    "# from jinxif import thresh\n",
    "# from jinxif import util\n",
    "\n",
    "from mplexable import _version\n",
    "from mplexable import afsub\n",
    "from mplexable import basic\n",
    "from mplexable import config\n",
    "from mplexable import feat\n",
    "from mplexable import imgmeta\n",
    "from mplexable import ometiff\n",
    "from mplexable import regist\n",
    "from mplexable import sane\n",
    "from mplexable import segment\n",
    "from mplexable import thresh\n",
    "from mplexable import util\n",
    "\n",
    "print('you are running jinxif version:', _version.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22509f79-4168-40b6-8c95-720a9832230c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9a92d01-01f4-44d9-a28d-49e740f8816e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set constants\n",
    "s_batch = 'RS-mTMA-20210707'\n",
    "ls_slide = [\n",
    "    'RS-mTMA-5',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b27f9a2f-3cac-4751-9d9d-ca0e1d25e823",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "464db05c-aeb4-4d0c-81d6-e8f45f9458b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# utility to handle file naming convention\n",
    "\n",
    "# this step is specific for the chinlab workflow.\n",
    "# by naming convention (specified in ~/.jinxif/config.py) the marker from all channels other than DAPI\n",
    "# (which is always in channel 1) are listed between the first and second underscore and separated by dots.\n",
    "# unfortunately, the software we use to generate the raw image tiff files from the microscopes czi files \n",
    "# (Zeiss Zen Blue) turns dots into underscores.\n",
    "# this function call will generate a \"rename dictionary\" that can be used as input for util.dchange_fname, \n",
    "# to rename the tiff files back to the specified naming convention standard.  \n",
    "\n",
    "util.underscore_to_dot(\n",
    "    es_slide = ls_slide,\n",
    "    s_start = 'R',\n",
    "    ei_underscore_to_dot = {1,2,3},\n",
    "    s_end = 'ORG.tif',\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/',\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/',  # s_rawdir, s_slide\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c9a2ea3-756d-43be-84f1-a3f13842fc3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# utility to rename files to be naming convention conform (specified in ~/.jinxif/config.py) \n",
    "\n",
    "# define renaming rules\n",
    "ds_rename = {\n",
    "    '_oldstring_':'_newstring_'\n",
    "}\n",
    "\n",
    "# run renaming function first with b_test=True (default setting). check output!\n",
    "# if you are sure that the function does what you want, set b_test=False (uncomment) and re-run the function.\n",
    "# comment out b_test again, so that  b_test=True (default setting) is activated again.\n",
    "\n",
    "for s_slide in ls_slide:\n",
    "    util.dchange_fname(\n",
    "        ds_rename = ds_rename,\n",
    "        #b_test = False,\n",
    "        s_wd = config.d_nconv['s_rawdir'] + f'{s_slide}/',  # 'RawImages/', # s_slide\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1b35f63-7a7b-453d-a8b4-3887ce4c4bef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e450cae6-0d68-433d-a5fa-59bcb87d77d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 1.1 check original czi image file names\n",
    "\n",
    "# this functions help to check the original czi folder for completeness of czi image files.\n",
    "# thereby, the file naming convention (specified in ~/.jinxif/config.py) is enforced.\n",
    "# additionally, marker names will be checked against correct labeling, as specified in ~/.jinxif/config.py.\n",
    "# at least one original czi is required and used for scene position metadata extraction. \n",
    "# original czi files might be used for exposure time metadata extraction.\n",
    "\n",
    "# run function and check output!\n",
    "# if filenames or markers are non-conform, correct them manually.\n",
    "# if markers are entirely missing, specify them in ~/.jinxif/config.py\n",
    "\n",
    "for s_slide in ls_slide:\n",
    "    df_img_slide = basic.parse_czi_original(\n",
    "        s_wd = config.d_nconv['s_format_czidir_original'].format(config.d_nconv['s_czidir'], s_slide),  #'{}{}/original/',  # s_czidir, s_slide\n",
    "    )\n",
    "    sane.count_images(df_img = df_img_slide)\n",
    "    sane.check_markers(\n",
    "        df_img = df_img_slide, \n",
    "        es_markerdapiblank_standard = config.es_markerdapiblank_standard,\n",
    "        es_markerpartition_standard = config.es_markerpartition_standard,\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3771ae07-409f-4354-93b8-3207e946666f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 1.2 check splitscene czi image file names\n",
    "\n",
    "# this functions help to check the splitscenes czi folder for completeness of czi image files.\n",
    "# thereby, the file naming convention (specified in ~/.jinxif/config.py) is enforced.\n",
    "# additionally, marker names will be checked against correct labeling, as specified in ~/.jinxif/config.py.\n",
    "# splitscenes czi files might be used for exposure time metadata extraction.\n",
    "# note: splitscenes czi files can not be used for scene position metadata extraction!\n",
    "\n",
    "# run function and check output!\n",
    "# if filenames or markers are non-conform, correct them manually.\n",
    "# if markers are entirely missing, specify them in ~/.jinxif/config.py\n",
    "\n",
    "for s_slide in ls_slide:\n",
    "    df_img_slide = basic.parse_czi_splitscene(\n",
    "        s_wd = config.d_nconv['s_format_czidir_splitscene'].format(config.d_nconv['s_czidir'], s_slide),  #'{}{}/splitscenes/',  # s_czidir, s_slide\n",
    "    )\n",
    "    sane.count_images(df_img = df_img_slide)\n",
    "    sane.check_markers(\n",
    "        df_img = df_img_slide, \n",
    "        es_markerdapiblank_standard = config.es_markerdapiblank_standard,\n",
    "        es_markerpartition_standard = config.es_markerpartition_standard,\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85569afd-8ef1-45a1-a917-884be3a2582a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 1.3 check raw tiff image file names\n",
    "\n",
    "# this functions help to check the raw tiff folder for completeness of tiff image files.\n",
    "# thereby, the file naming convention (specified in ~/.jinxif/config.py) is enforced.\n",
    "# additionally, marker names will be checked against correct labeling, as specified in ~/.jinxif/config.py.\n",
    "# note: raw tiff files can not be used for exposure time or scene position metadata extraction!\n",
    "\n",
    "# run function and check output!\n",
    "# if filenames or markers are non-conform, \n",
    "# correct them with the utility util.dchange_fname function in the jupyter cell above.\n",
    "# if markers are entirely missing, specify them in ~/.jinxif/config.py\n",
    "\n",
    "for s_slide in ls_slide:\n",
    "    print(f'process slide: {s_slide} ...')\n",
    "    df_img_slide = basic.parse_tiff_raw(\n",
    "        s_wd = config.d_nconv['s_format_rawdir'].format(config.d_nconv['s_rawdir'], s_slide),  # 'RawImages/{}/', # s_slide\n",
    "    )\n",
    "    sane.count_images(df_img = df_img_slide)\n",
    "    sane.check_markers(\n",
    "        df_img = df_img_slide, \n",
    "        es_markerdapiblank_standard = config.es_markerdapiblank_standard,\n",
    "        es_markerpartition_standard = config.es_markerpartition_standard,\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa6fa4d-ecad-4e03-912f-8685def5711e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.4 generate qc images from the raw tiff images\n",
    "\n",
    "# at s_qcdir check the generated qc png images! \n",
    "# fix whatever is necessary\n",
    "\n",
    "sane.visualize_raw_images_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    s_color = config.d_nconv['s_color_dapi_jinxif'],  #'c1',\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/'\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/', # s_rawdir, s_slide\n",
    "    s_qcdir = config.d_nconv['s_qcdir'],  #'QC/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95504231-0159-47b5-8b43-7994642dd96c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65e767b0-f339-4fb5-a85c-c9c4fed27214",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 2.0 extract image metadata\n",
    "\n",
    "# the metadata is by default extracted from the original czi files.\n",
    "# alternatively, exposure time can be extracted from the splitscenes czi files (by setting b_exposuretime_splitscene to True).\n",
    "# with default naming convention, the original files are expected to be under {s_czidir}/{s_slide_id}/original/\n",
    "# and the splitscenes czi files are expected to be under {s_czidir}/{s_slide_id}/splitscenes/\n",
    "# Additionaly, the czi images filenames them self have to follow the correct naming convention.\n",
    "\n",
    "imgmeta.fetch_meta_batch(\n",
    "    es_slide = ls_slide,\n",
    "    s_czidir = config.d_nconv['s_czidir'], # 'CziImages/',\n",
    "    s_format_czidir_original = config.d_nconv['s_format_czidir_original'], #'{}{}/original/',  # s_czidir, s_slide\n",
    "    s_format_czidir_splitscene = config.d_nconv['s_format_czidir_splitscene'], #'{}{}/splitscenes/',  # s_czidir, s_slide\n",
    "    s_sceneposition_round = 'R1_',  # file matching pattern, can but does not have to be round.  \n",
    "    b_exposuretime_original = True,\n",
    "    b_exposuretime_splitscene = True,\n",
    "    b_sceneposition_original = True,\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c13c455-2499-42d7-80a8-3d56b20e243e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.1 TMA layout (optional)\n",
    "\n",
    "# for each slide that is a tissue microarray (TMA), \n",
    "# run util.tma_grid to layout the TMA, based on the extracted scene position information.\n",
    "# utilize {s_metadir}/{s_slide}_ScenePositions_coor_{r_sampler}.png \n",
    "# and {s_metadir}/{s_slide}_ScenePositions_coor.csv to map alphanumeric coordinate labels (scene_coor column).\n",
    "# if necessary, in {s_metadir}/{s_slide}_ScenePositions_coor.csv the scene_coor can manually be changed.\n",
    "# when editing {s_metadir}/{s_slide}_ScenePositions_coor.csv, if needed, additional columns can be added. \n",
    "# when done, rename \n",
    "# {s_metadir}/{s_slide}_ScenePositions_coor.csv to\n",
    "# {s_metadir}/{s_slide}_ScenePositions_coor_ok.csv\n",
    "# so, that the work is not overwritten when util.tma_grid is run again.\n",
    "\n",
    "for s_slide, i_core_yaxis, i_core_xaxis in [\n",
    "       ['slide-id-tma', None, None],\n",
    "    ]:\n",
    "    util.tma_grid(\n",
    "        s_slide = s_slide,\n",
    "        i_core_yaxis = i_core_yaxis,\n",
    "        i_core_xaxis = i_core_xaxis,\n",
    "        r_sampler = 0.98,\n",
    "        s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ecabfee-f3c8-456f-bf54-a17fd57885f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b0c945c-a74a-4c9e-85b8-80605f591809",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.1 utility for template ddd_crop (helper function)\n",
    " \n",
    "# run this function to generate a template ddd_crop dictionary.\n",
    "# this template ddd_crop dictionaries are helpful to write the ddd_crop specification at 3.3.\n",
    "\n",
    "util.template_dddcrop(\n",
    "    ls_slide = [\n",
    "        'slide-id',\n",
    "    ],\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/',\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/',  # s_rawdir, s_slide  # to get slide_mscene\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    "    s_format_metafile_tmacoorcsv = '{}{}_ScenePositions_coor_ok.csv',  # s_metadir, s_slide  # for alphanumeric TMA coordinates!\n",
    "    s_type_coor = 'xywh',  # xywh xyxy yxyx None\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "941a65f8-f604-474e-8c12-e981eb776973",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.2 utility for grid cropping (helper function)\n",
    "\n",
    "# run this function for huge tissue slide_mscenes, that should be grid cropped to smaller slide_pxscenes.\n",
    "# the d_crop dictionary output can be used to write the ddd_crop specification at 3.3.\n",
    "\n",
    "util.gridcrop(\n",
    "    li_xywh = [0,0, 30000,30000],  # whole tissue square crop coordinates. the li_xywh coordinates you can get with irfanview or similar software.\n",
    "    i_max = 20000,  # no cropped rectangle will be wider or higher than i_max pixel!\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6765cd55-85e6-4342-a901-520178c0d6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.3  specify ddd_crop\n",
    "\n",
    "# this variable is needed for registration and down stream!\n",
    "# every slide mscene have to be mapped to one or more pxscene(s).\n",
    "# if no cropping at all is requited, [0,0, 0,0, 'nnnn'] can be replaced by None.\n",
    "# you can use the utilities at 3.1 and 3.2 to help you to specify this variable.\n",
    "\n",
    "# run!\n",
    "ddd_crop = {\n",
    "    'RS-mTMA-5': {\n",
    "        'Scene-A01': {'sceneA01': [426,880,6000,6000, 'xywh']},\n",
    "        'Scene-A04': {'sceneA04': [136,2373,6000,3000, 'xywh']},\n",
    "        'Scene-A05': {'sceneA05': [315,2614,6000,4000, 'xywh']},\n",
    "        'Scene-A06': {'sceneA06': [45,1007,3000,3000, 'xywh']},\n",
    "        'Scene-A07': {'sceneA07': [754,424,6000,6000, 'xywh']},\n",
    "        'Scene-A08': {'sceneA08': [397,934,6000,6000, 'xywh']},\n",
    "        'Scene-A09': {'sceneA09': [357,502,6000,6000, 'xywh']},\n",
    "        'Scene-A11': {'sceneA11': [317,338,6000,6000, 'xywh']},\n",
    "        'Scene-B01': {'sceneB01': [277,1109,6000,6000, 'xywh']},\n",
    "        'Scene-B04': {'sceneB04': [127,158,5500,5500, 'xywh']},\n",
    "        'Scene-B05': {'sceneB05': [568,1204,6000,6000, 'xywh']},\n",
    "        'Scene-B06': {'sceneB06': [1374,565,6000,6000, 'xywh']},\n",
    "        'Scene-B07': {'sceneB07': [674,442,6000,6000, 'xywh']},\n",
    "        'Scene-B08': {'sceneB08': [1260,1502,6000,6000, 'xywh']},\n",
    "        'Scene-B09': {'sceneB09': [560,850,6000,6000, 'xywh']},\n",
    "        'Scene-B10': {'sceneB10': [889,877,6000,4000, 'xywh']},\n",
    "        'Scene-B11': {'sceneB11': [842,740,6000,6000, 'xywh']},\n",
    "        'Scene-C01': {'sceneC01': [20,14,5500,5500, 'xywh']},\n",
    "        'Scene-C03': {'sceneC03': [337,331,6000,6000, 'xywh']},\n",
    "        'Scene-C04': {'sceneC04': [42,222,6000,6000, 'xywh']},\n",
    "        'Scene-C05': {'sceneC05': [1115,51,5500,5500, 'xywh']},\n",
    "        'Scene-C06': {'sceneC06': [18,21,5500,5500, 'xywh']},\n",
    "        'Scene-C07': {'sceneC07': [965,746,6000,6000, 'xywh']},\n",
    "        'Scene-C08': {'sceneC08': [180,126,5500,5500, 'xywh']},\n",
    "        'Scene-C09': {'sceneC09': [918,1072,6000,6000, 'xywh']},\n",
    "        'Scene-C10': {'sceneC10': [64,756,5500,6000, 'xywh']},\n",
    "        'Scene-C11': {'sceneC11': [1165,743,6000,6000, 'xywh']},\n",
    "        'Scene-C12': {'sceneC12': [141,772,5500,6000, 'xywh']},\n",
    "        'Scene-D01': {'sceneD01': [1193,29,5500,5500, 'xywh']},\n",
    "        'Scene-D03': {'sceneD03': [1552,436,6000,6000, 'xywh']},\n",
    "        'Scene-D04': {'sceneD04': [27,231,5000,6000, 'xywh']},\n",
    "        'Scene-D05': {'sceneD05': [1211,425,6000,6000, 'xywh']},\n",
    "        'Scene-D06': {'sceneD06': [531,1162,6000,6000, 'xywh']},\n",
    "        'Scene-D07': {'sceneD07': [229,1046,6000,6000, 'xywh']},\n",
    "        'Scene-D08': {'sceneD08': [157,194,6000,5500, 'xywh']},\n",
    "        'Scene-D09': {'sceneD09': [746,28,6000,5500, 'xywh']},\n",
    "        'Scene-D10': {'sceneD10': [25,1143,6000,6000, 'xywh']},  # missing!\n",
    "        'Scene-D11': {'sceneD11': [1332,1061,6000,6000, 'xywh']},\n",
    "        'Scene-E02': {'sceneE02': [762,306,6000,6000, 'xywh']},\n",
    "        'Scene-E03': {'sceneE03': [110,321,6000,6000, 'xywh']},\n",
    "        'Scene-E04': {'sceneE04': [998,550,6000,6000, 'xywh']},\n",
    "        'Scene-E05': {'sceneE05': [1563,1519,6000,6000, 'xywh']},\n",
    "        'Scene-E06': {'sceneE06': [808,1369,6000,6000, 'xywh']},\n",
    "        'Scene-E07': {'sceneE07': [46,1150,6000,6000, 'xywh']},\n",
    "        'Scene-E08': {'sceneE08': [188,84,5500,5500, 'xywh']},\n",
    "        'Scene-E09': {'sceneE09': [959,1137,6000,6000, 'xywh']},\n",
    "        'Scene-E10': {'sceneE10': [39,19,6000,5500, 'xywh']},\n",
    "        'Scene-E11': {'sceneE11': [19,20,5500,5500, 'xywh']},\n",
    "        'Scene-E12': {'sceneE12': [446,942,6000,6000, 'xywh']},\n",
    "        'Scene-F02': {'sceneF02': [1389,1085,6000,6000, 'xywh']},\n",
    "        'Scene-F03': {'sceneF03': [874,13,6500,6500, 'xywh']},\n",
    "        'Scene-F04': {'sceneF04': [645,11,3000,3000, 'xywh']},\n",
    "        'Scene-F05': {'sceneF05': [12,719,6000,6000, 'xywh']},\n",
    "        'Scene-F07': {'sceneF07': [371,1415,5000,5000, 'xywh']},\n",
    "        'Scene-F08': {'sceneF08': [861,136,6000,5500, 'xywh']},\n",
    "        'Scene-F09': {'sceneF09': [1027,880,6000,6000, 'xywh']},\n",
    "        'Scene-F10': {'sceneF10': [352,153,6000,5500, 'xywh']},\n",
    "        #'Scene-F11': {'sceneF11': [1373,497,5500,4500, 'xywh']},  # dropped! tissue lost and does not register.\n",
    "        'Scene-F12': {'sceneF12': [682,123,6000,5500, 'xywh']},\n",
    "        'Scene-G01': {'sceneG01': [58,419,6000,6000, 'xywh']},\n",
    "        'Scene-G02': {'sceneG02': [946,564,6000,6000, 'xywh']},\n",
    "        'Scene-G04': {'sceneG04': [445,1528,6000,6000, 'xywh']},\n",
    "        'Scene-G05': {'sceneG05': [1250,979,6000,6000, 'xywh']},\n",
    "        'Scene-G06': {'sceneG06': [185,616,6000,6000, 'xywh']},\n",
    "        'Scene-G08': {'sceneG08': [667,301,6000,6000, 'xywh']},\n",
    "        'Scene-G09': {'sceneG09': [991,673,6000,6000, 'xywh']},\n",
    "        'Scene-G10': {'sceneG10': [1018,195,6000,5500, 'xywh']},\n",
    "        'Scene-G11': {'sceneG11': [4,206,5500,5500, 'xywh']},\n",
    "        'Scene-G12': {'sceneG12': [583,317,6000,6000, 'xywh']},\n",
    "        'Scene-H03': {'sceneH03': [1110,79,6000,5500, 'xywh']},\n",
    "        'Scene-H04': {'sceneH04': [622,75,6000,5000, 'xywh']},\n",
    "        'Scene-H05': {'sceneH05': [321,581,5000,5000, 'xywh']},\n",
    "        'Scene-H06': {'sceneH06': [776,850,6000,5000, 'xywh']},\n",
    "        'Scene-H07': {'sceneH07': [144,879,5500,6000, 'xywh']},\n",
    "        'Scene-H08': {'sceneH08': [20,203,5500,5500, 'xywh']},\n",
    "        'Scene-H09': {'sceneH09': [915,1050,5500,5500, 'xywh']},\n",
    "        'Scene-H10': {'sceneH10': [758,790,6000,6000, 'xywh']},\n",
    "        'Scene-H11': {'sceneH11': [1474,578,6000,6000, 'xywh']},\n",
    "        'Scene-H12': {'sceneH12': [732,840,6000,6000, 'xywh']},\n",
    "        'Scene-I01': {'sceneI01': [1056,23,6000,6000, 'xywh']},\n",
    "        'Scene-I02': {'sceneI02': [218,1152,5500,6000, 'xywh']},\n",
    "        'Scene-I03': {'sceneI03': [1389,20,6000,6000, 'xywh']},\n",
    "        'Scene-I04': {'sceneI04': [214,1273,5500,6000, 'xywh']},\n",
    "        'Scene-I05': {'sceneI05': [16,628,5500,6000, 'xywh']},\n",
    "        'Scene-I06': {'sceneI06': [849,24,6000,6000, 'xywh']},\n",
    "        'Scene-I07': {'sceneI07': [585,129,6000,5500, 'xywh']},\n",
    "        'Scene-I08': {'sceneI08': [1019,27,6000,6000, 'xywh']},\n",
    "        'Scene-I09': {'sceneI09': [1013,564,6000,6000, 'xywh']},\n",
    "        'Scene-I10': {'sceneI10': [210,909,5500,6000, 'xywh']},\n",
    "        'Scene-I11': {'sceneI11': [25,346,6000,6000, 'xywh']},\n",
    "        'Scene-I12': {'sceneI12': [562,1007,6000,6000, 'xywh']},\n",
    "    },\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58a5c2b4-c9f4-4d65-9232-b2ff5789a27a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.4 write ddd_crop dictionary to file\n",
    "\n",
    "# this is just for backup, to save your work done at 3.3.\n",
    "# result json file will be stored in s_rawdir.\n",
    "\n",
    "regist.save_cropcoor(\n",
    "    s_batch = s_batch,\n",
    "    ddd_crop = ddd_crop,\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcea6abf-28cd-4bd5-bb40-a18673de1011",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17267381-013c-431e-9e61-d522d3fec59c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 4.0 register images \n",
    "\n",
    "# register and possibly crop each slide specified in es_slide.\n",
    "# every slide in es_slide have to be specified in ddd_crop.\n",
    "\n",
    "regist.regist_spawn(\n",
    "    ddd_crop = ddd_crop,\n",
    "    es_slide = ls_slide,\n",
    "    s_type_registration = 'matlab',\n",
    "    # file extension\n",
    "    s_regex_ext = r'_(ORG.tif)$',  # regex file extension.\n",
    "    # staining round\n",
    "    s_regex_round_ref = r'^(R\\d+Q?_).+$',  # regex round of raw reference round tiffs.\n",
    "    s_regex_round_nonref = r'^(R\\d+Q?_).+$',  # regex round of raw non-reference round tiffs.\n",
    "    # staining marker\n",
    "    s_regex_marker_ref = r'^.+_(.+\\..+\\..+\\.[^_]+)_.+$',  # regex of reference round markers in raw tiffs.\n",
    "    s_regex_marker_nonref = r'^.+_(.+\\..+\\..+\\.[^_]+)_.+$',  # regex of non-reference round markers in raw tiffs.\n",
    "    # microscopy channel\n",
    "    s_regex_micchannel_ref = r'^.*_(c\\d+)_.*$',  # regex of reference round microscopy channels/colors.\n",
    "    s_regex_micchannel_nonref = r'^.*_(c\\d+)_.*$',  # regex of non-reference round microscopy channels/colors.\n",
    "    # dapi images only\n",
    "    s_glob_img_dapiref = 'R1_*_{}_*-{}_c1_ORG.tif', # glob pattern of raw dapi exclusive reference round tiff, fetching slide and microscopy scene id.\n",
    "    s_glob_img_dapinonref = 'R*_*_{}_*-{}_c1_ORG.tif', # glob pattern of raw dapi non-reference round tiffs, fetching slide and microscopy scene id.\n",
    "    # non-dapi images (possibly dapi images too)\n",
    "    s_glob_img_ref = 'R1_*_{}_*-{}_c*_ORG.tif', # glob pattern of raw exclusive reference round tiffs, fetching slide and microscopy scene id.\n",
    "    s_glob_img_nonref = 'R*_*_{}_*-{}_c*_ORG.tif', # glob pattern of raw non-reference round tiffs, fetching slide and microscopy scene id.\n",
    "    # registration\n",
    "    i_npoint = str(10000),  # number of key points used for registration,\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # filesystem\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/',\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/',  # s_rawdir, s_slide\n",
    "    s_qcregistration_dir = 'QC/RegistrationPlots/',\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3556b357-8c73-450b-a84b-7ea876bdeed0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4.1 qc plots from registered images\n",
    "\n",
    "# generation qc png plots from the dapi channel registered tiff images, and possibly other channels.\n",
    "# check generated qc png plots in the s_qcdir/s_regdir/ folder.\n",
    "# fix whatever is necessary.\n",
    "\n",
    "regist.visualize_reg_images_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    s_color = config.d_nconv['s_color_dapi_jinxif'],  #'c1'\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '32G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    "    s_format_regdir = config.d_nconv['s_format_regdir'],  #'{}{}/', # s_regdir, s_slide_pxscene\n",
    "    s_qcdir = config.d_nconv['s_qcdir'],  #'QC/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2397538b-b758-484f-b67b-0c87e41f23a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa1afd3e-1f4c-446c-8e76-2ed1f8cdcc14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.0 exposure qc\n",
    "\n",
    "# generate a batch wide exposure time png and csv matrix.\n",
    "\n",
    "# check s_metadir/s_batch_exposure_time_ms_matrix.png and \n",
    "# s_metadir/s_batch_exposure_time_ms_matrix.csv for wrong exposure time values. \n",
    "# in the csv, the marker exposure time mean value in the far right column,\n",
    "# and the per slide or slide_scene summed up exposure time value at the bottom row \n",
    "# are very helpful to find errors.\n",
    "\n",
    "# if you find errors, specify ddd_ect at 5.2 how to do exposure time correction! \n",
    "\n",
    "imgmeta.exposure_matrix(\n",
    "    s_batch = s_batch,\n",
    "    es_slide = ls_slide, \n",
    "    tr_figsize = (32,20),  # (w,h)\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de9b8311-a3f5-4acc-9ef6-c58774d94b3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.1 utility for template ddd_etc (helper function)\n",
    "\n",
    "# specify all slide_pxscene that have to be exposure time corrected.\n",
    "\n",
    "util.template_dddetc(\n",
    "    ls_slidepxscene = [\n",
    "    ],\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec120e5e-59a1-4a44-bafd-894c32f6d05b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.2 specify  ddd_etc\n",
    "\n",
    "# empty dictionary, if nothing to correct.\n",
    "# else, the dictionary format should look like this:\n",
    "# ddd_etc = {'slide_scene': {'marker': {'is': 7,'should_be': 4}},}\n",
    "\n",
    "# bue 20210120: since this is batch has only one slide, it is hard to detect error on slide level. \n",
    "# above all, cores have the same exposure time. \n",
    "# no et correvtion needed!\n",
    "\n",
    "# run!\n",
    "ddd_etc = {\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18c32389-0134-42d6-a548-e1d5f56159d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.3 write ddd_etc dictionary to file\n",
    "\n",
    "# this is just for backup, to save your work done at 5.2. \n",
    "# result json file will be stored in s_metadir.\n",
    "\n",
    "regist.save_exposuretimecorrect(\n",
    "    s_batch =  s_batch,\n",
    "    ddd_etc = ddd_etc,\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fe0313e-5273-4772-b338-56c9062a9a10",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5.4 exposure time correct registered images\n",
    "\n",
    "# run, if necessary.\n",
    "\n",
    "regist.exposure_time_correct_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    ddd_etc = ddd_etc,\n",
    "    s_imagetype_original = 'ORG',\n",
    "    # processing \n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '32G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    "    s_format_regdir = config.d_nconv['s_format_regdir'],  #'{}{}/',  s_regdir, s_slide_pxscene\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4fbe48f-2ff5-4790-84bd-7fbddaa98373",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdc49ad5-2e8c-4877-a476-59ea9d59d635",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6.0 generate marker table\n",
    "\n",
    "# the whole batch, every slide_pxscene, will have the same marker table.\n",
    "# for s_slide_pxscene choose one slide_pxscene to generate the marker table.\n",
    "\n",
    "# result csv will be stored in s_regdir.\n",
    "\n",
    "# fetch slide_pxscene from ddd_crop\n",
    "s_slide = sorted(ddd_crop.keys())[0] \n",
    "s_mscene =  sorted(ddd_crop[s_slide].keys())[0]\n",
    "s_pxscene =  sorted(ddd_crop[s_slide][s_mscene].keys())[0]\n",
    "s_slide_pxscene = s_slide + '_' + s_pxscene\n",
    "\n",
    "# generate tabel\n",
    "basic.marker_table(\n",
    "    s_slide_pxscene = s_slide_pxscene,  # this is a slide_pxscene defined in ddd_crop above\n",
    "    s_batch = s_batch,\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    "    s_format_regdir = config.d_nconv['s_format_regdir'],  #'{}{}/',  # s_regdir, s_slide_pxscene\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4473504-c4a8-44d6-b287-bed48281e859",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35abb73b-96c3-4687-ba9a-4978a41af5ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7.0 auto fluorescent subtract images\n",
    "\n",
    "# specify the quenching round markers in ds_early and ds_late according to the marker tabel displayed at 6.0.\n",
    "# if there is only one quenching round, use empty dictionary for ds_early.\n",
    "# if there is no quenching round, you can not do auto fluorescent subtraction.\n",
    "\n",
    "afsub.afsub_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    ddd_crop = ddd_crop,\n",
    "    ddd_etc = ddd_etc,\n",
    "    ds_early = {'c2':'R0c2','c3':'R0c3','c4':'R0c4','c5':'R0c5'},  # adjust accoring to the marker table. \n",
    "    ds_late = {'c2':'R5Qc2','c3':'R5Qc3','c4':'R5Qc4','c5':'R5Qc5'},   # adjust accoring to the marker table. \n",
    "    es_exclude_color = {'c1','c5'},\n",
    "    es_exclude_marker = {},\n",
    "    b_8bit = False,\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    "    s_format_regdir = config.d_nconv['s_format_regdir'],  #'{}{}/', # s_regdir, s_slide_pxscene\n",
    "    s_afsubdir = config.d_nconv['s_afsubdir'],  #'SubtractedRegisteredImages/',\n",
    "    s_format_afsubdir = config.d_nconv['s_format_afsubdir'],  #'{}{}/', # s_afsubdir, s_slide_pxscene\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "631365e5-0a1c-4709-8f8e-88e125c0b0a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7.1 qc plots from auto fluorescent subtracted registered images (optional)\n",
    "\n",
    "# generation qc png plots from the af subtracted registered images.\n",
    "\n",
    "for s_color in ['c1','c2','c3','c4','c5']:\n",
    "    regist.visualize_reg_images_spawn(\n",
    "        es_slide = ls_slide,\n",
    "        s_color = s_color,\n",
    "        # processing\n",
    "        s_type_processing = 'slurm',\n",
    "        s_slurm_partition = 'exacloud',\n",
    "        s_slurm_mem = '36G',\n",
    "        s_slurm_time = '36:00:0',\n",
    "        s_slurm_account = 'gray_lab',\n",
    "        # file system\n",
    "        s_regdir = config.d_nconv['s_afsubdir'],  #'RegisteredImages/'\n",
    "        s_format_regdir = config.d_nconv['s_format_afsubdir'],  #'{}{}/',  # s_afsubdir, s_slide_pxscene\n",
    "        s_qcdir = config.d_nconv['s_qcdir'],  #'QC/'\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76080e9f-dc1f-4360-aa00-d80ab35eff79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f54ecde-355f-483d-91e7-dd6205e33546",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 8.0 segmentation\n",
    "\n",
    "# run s_task nuc or cell (or nuccell) for nucleus and/or cell segmentation.\n",
    "# output nuc segmentation: s_segdir/s_slide_CellposeSegmentation/{slide_scene}_nuc{nuc_diam}_NucleiSegmentationBasins.tif\n",
    "# output cell segmentation s_segdir/s_slide_CellposeSegmentation/{slide_scene}_{seg_marker}_nuc{nuc_diam}_cell{cell_diam}_matched_CellSegmentationBasins.tif\n",
    "\n",
    "# then run s_task match to match nucleus and cell labels.\n",
    "# output cell segmentation s_segdir/s_slide_CellposeSegmentation/{slide_scene}_{seg_marker}_cell{nuc_diam}_CellSegmentationBasins.tif\n",
    "\n",
    "segment.segment_spawn(\n",
    "    # input\n",
    "    es_slide = ls_slide,\n",
    "    s_task = 'match', # known segmentation tasks are nuc, cell, or nuccell, and match.\n",
    "    # segmentation\n",
    "    i_nuc_diam = 30,  # numer of pixel for minimal nucleus diameter. microscope dependent!\n",
    "    i_cell_diam = 30,  # numer of pixel for minimal cell diameter. microscope dependent!\n",
    "    s_dapi_round = 'R1',  # the same round that images were registered to.\n",
    "    es_seg_marker = {'Ecad'},  # {'Ecad'}, # specify the cell segmentation marker(s). if None, no cell segmentation and label matching will be done.\n",
    "    es_rare_marker = set(),  # markers will be slightly enhanced.\n",
    "    s_type_data = 'cmif',  # implemented are 'cmif' and  'codex'.\n",
    "    # gpu\n",
    "    s_gpu = 'gpu:v100:1',  # if None, CPU will be used. anything else will use GPU. if s_type_processing is 'slurm', s_gpu will be used in slurm command call.\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem ='128G',\n",
    "    s_slurm_time ='12:00:00',\n",
    "    s_slurm_account ='gray_lab',\n",
    "    # file system\n",
    "    s_regdir = config.d_nconv['s_regdir'],  #'RegisteredImages/',\n",
    "    s_format_regdir = config.d_nconv['s_format_regdir'],  #'{}{}/', # s_regdir, s_slide_pxscene\n",
    "    s_segdir = config.d_nconv['s_segdir'],  #'Segmentation/',\n",
    "    s_format_segdir_cellpose = config.d_nconv['s_format_segdir_cellpose'],  #'{}{}_CellposeSegmentation/', # s_segdir, s_slide\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5609353a-45d8-4827-a748-602753e69bae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe5e24cf-dc89-4d08-830d-24710d587530",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 9.0 auto threshold markers\n",
    "\n",
    "# loop will process registered, and autofluorescent subtracted registered images.\n",
    "# result csv will be stored in s_regdir and s_afsubdir, respective.\n",
    "\n",
    "for s_inputdir, s_format_inputdir in [\n",
    "        (config.d_nconv['s_regdir'], config.d_nconv['s_format_regdir']),\n",
    "        (config.d_nconv['s_afsubdir'], config.d_nconv['s_format_afsubdir']),\n",
    "    ]:\n",
    "    thresh.auto_thresh_spawn(\n",
    "        es_slide = ls_slide,\n",
    "        # processing\n",
    "        s_type_processing = 'slurm',\n",
    "        s_slurm_partition = 'exacloud',\n",
    "        s_slurm_mem = '32G',\n",
    "        s_slurm_time = '36:00:00',\n",
    "        s_slurm_account = 'gray_lab',\n",
    "        # file system\n",
    "        s_afsubdir = s_inputdir,  #'SubtractedRegisteredImages/' 'RegisteredImages/'\n",
    "        s_format_afsubdir = s_format_inputdir,  #'{}{}/' # s_afsubdir, s_slide_pxscene\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eb89fef-5de5-485c-9892-8634bb93e2f4",
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27650180-875d-49d9-8764-9cfdbbd2a5a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10.0 feature extraction\n",
    "\n",
    "# loop will process registered, and autofluorescent subtracted registered images.\n",
    "\n",
    "# result csv will be stored in s_segdir/s_slide_CellposeSegmentation/\n",
    "# output registered images: features_{slide_scene}_{seg_marker}_raw_MeanIntensity_Shape_Centroid_registeredimages.csv\n",
    "# output af subtracted registered images: features_{slide_scene}_{seg_marker}_raw_MeanIntensity_Shape_Centroid_subtractedregisteredimages.csv\n",
    "\n",
    "for s_inputdir, s_format_inputdir in [\n",
    "        (config.d_nconv['s_regdir'], config.d_nconv['s_format_regdir']),\n",
    "        (config.d_nconv['s_afsubdir'], config.d_nconv['s_format_afsubdir']),\n",
    "    ]:\n",
    "    feat.extract_features_spawn(\n",
    "        es_slide = ls_slide,\n",
    "        es_seg_marker = {'Ecad'},  # {'Ecad'}, # use the same cell segmentation marker(s) specified at 8.0. \n",
    "        i_exp = 5,  # number of pixel for cytoplasm doughnut. microscope dependent!\n",
    "        i_mem = 2,  # number of pixel for membrane. microscope dependent!\n",
    "        i_shrink = 0,  # optional, e.g. for patching against bleed through at 10.1 by setting es_custom_markerpartition, or es_shrink_marker and i_shrink.\n",
    "        # processing\n",
    "        s_type_processing = 'slurm',\n",
    "        s_slurm_partition = 'exacloud',\n",
    "        s_slurm_mem = '256G',  # this function eats RAM like no other!\n",
    "        s_slurm_time = '36:00:00',\n",
    "        s_slurm_account = 'gray_lab',\n",
    "        # specify input and output directory\n",
    "        s_afsubdir = s_inputdir,  #'RegisteredImages/', or 'SubtractedRegisteredImages/',\n",
    "        s_format_afsubdir = s_format_inputdir,  #'{}{}/' # s_afsubdir, s_slide_pxscene\n",
    "        s_segdir = config.d_nconv['s_segdir'],  #'Segmentation/',\n",
    "        s_format_segdir_cellpose = config.d_nconv['s_format_segdir_cellpose'],\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f0f75b8-65af-4acd-be2e-09c9d4bcca05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10.1 feature filtering and patching\n",
    "\n",
    "# loop will process registered, and autofluorescent subtracted registered images.\n",
    "# important: depends on in which channel s_thresh_marker was, i_thresh_manual value might not be independent of autofluorescent subtraction.\n",
    "\n",
    "# result csv will be stored in s_segdir/s_slide_CellposeSegmentation/:\n",
    "# features_{slide}_CentroidXY.csv\n",
    "# features_{slide}_{s_thresh}_patched_MeanIntensity_Shape_{es_dapipartition_filter}_registeredimages.csv \n",
    "# features_{slide}_{s_thresh}_patched_MeanIntensity_Shape_{es_dapipartition_filter}_subtractedregisteredimages.csv\n",
    "\n",
    "# result png will be stored in s_qcdir/Segmentation/:\n",
    "# {slide}_DAPI_rounds_registeredimages_lineplot.png\n",
    "# {slide_scene}_DAPIn.DAPIm.DAPIo.DAPIp.{s_thresh}_thresh_registeredimages_scatter.png\n",
    "# {slide_scene}_DAPIn.DAPIm.DAPIo.DAPIp.{s_thresh}_thresh_subtractedregisteredimages_scatter.png\n",
    "\n",
    "# about the 3 related parameters, es_seg_marker, di_seg_marker, and des_cytoplasm_marker:\n",
    "# you can specify as many celltype specific segmentation markers in es_seg_marker (specified and used at 8.0 and used at 10.0) and di_seg_marker,  \n",
    "# as you have in the panel, and for which you have specified cytoplasm marker in des_cytoplasm_marker (specified at ~/.jinxif/config.py).\n",
    "# di_seg_marker.keys() has to be a member of es_seg_marker and des_cytoplasm_marker.keys().\n",
    "# the thing is, marker specified in des_cytoplasm_marker (values) will extract the mean value from the segmented cytoplasm, \n",
    "# instead of extracting the mean value from an i_exp thick doughnut around the nucleus.\n",
    "# if you're not specifying any marker at des_cytoplasm_marker, setting di_seg_marker will have no effect.\n",
    "\n",
    "# about di_seg_marker:\n",
    "# set to None, if no cell segmentation was done.\n",
    "# specify the cell segmentation marker (dictionary key) to be used to detect cytoplasm positive cells. this has to be a marker from es_seg_marker set specified at 8.0. \n",
    "# specify the as min threshold (dictionary value) for the segmentation marker. this value is microscope setting dependent!\n",
    "# read above about the 3 related parameters, es_seg_marker, di_seg_marker, and des_cytoplasm_marker!\n",
    "\n",
    "for s_inputdir, s_format_inputdir, di_seg_marker in [\n",
    "        (config.d_nconv['s_regdir'], config.d_nconv['s_format_regdir'], {'Ecad': 1000}),  # {'Ecad': 1000} # read above about di_seg_marker!\n",
    "        (config.d_nconv['s_afsubdir'], config.d_nconv['s_format_afsubdir'], {'Ecad': 1000}),  # {'Ecad': 1000} # read above about di_seg_marker!\n",
    "    ]:\n",
    "    feat.filter_features_spawn(\n",
    "        es_slide = ls_slide,\n",
    "        es_dapipartition_filter = {'DAPI1_nuclei','DAPI8_nuclei'},  # {'DAPI1_nuclei','DAPI16_nuclei'}, # for nuclei filtering, specify DAPI from the first and last round, and optionally week round in between.\n",
    "        di_seg_marker = di_seg_marker,  # {'Ecad': 1000}, # read above about di_seg_marker!\n",
    "        i_exp = 5,  # number of pixel for cytoplasm doughnut. microscope dependent!\n",
    "        i_mem = 2,  # number of pixel for membrane. microscope dependent!\n",
    "        i_shrink = 0,  # optional, for patching against bleed trough. use the same setting as in 10.0.\n",
    "        es_shrink_marker = set(),   # optional, specify marker to be patching against bleed through.\n",
    "        es_custom_markerpartition = set(),  # optional, for marker_partition in the raw feature extracted csv, that should additional be ported to the patched csv. \n",
    "        des_cytoplasm_marker = config.des_cytoplasmmarker_standard,  # read above about the 3 related parameters es_seg_marker, di_seg_marker, and des_cytoplasm_marker!\n",
    "        s_tissue_dapi = 'DAPI1',  # used dapi from the same round that images were registered to.\n",
    "        i_tissue_dapi_thresh = 500,  # 300 - 600,\n",
    "        i_tissue_area_thresh = 50000,  # 65536\n",
    "        ds_shape = {  # specify according to your dapi and s_thresh_marker. for shape determination, we use dapi from round 2 because it is less blurry than dapi from round 1.\n",
    "            'DAPI2_nuclei_area': 'nuclei_area',\n",
    "            'DAPI2_nuclei_eccentricity': 'nuclei_eccentricity',\n",
    "            'Ecad_cell_area': 'cell_area',\n",
    "            'Ecad_cell_eccentricity': 'cell_eccentricity',\n",
    "            'Ecad_cell_euler': 'cell_euler',\n",
    "            'Ecad_cytoplasm_area': 'cytoplasm_area',\n",
    "            'Ecad_cytoplasm_eccentricity': 'cytoplasm_eccentricity',\n",
    "            'Ecad_cytoplasm_euler': 'cytoplasm_euler',\n",
    "        },\n",
    "        ds_centroid = { # specify the dapi round for centroid coordinate detection. we use the same round that images were registered to.\n",
    "            'DAPI2_nuclei_centroid-0': 'DAPI_Y',\n",
    "            'DAPI2_nuclei_centroid-1': 'DAPI_X',\n",
    "        },\n",
    "        # processing\n",
    "        s_type_processing = 'slurm',\n",
    "        s_slurm_partition = 'exacloud',\n",
    "        s_slurm_mem = '64G',\n",
    "        s_slurm_time = '36:00:00',\n",
    "        s_slurm_account = 'gray_lab',\n",
    "        # filter\n",
    "        s_afsubdir = s_inputdir,  #'RegisteredImages/', or 'SubtractedRegisteredImages/',\n",
    "        s_format_afsubdir = s_format_inputdir,  # '{}{}/', # s_afsubdir, slide_scene\n",
    "        s_segdir = config.d_nconv['s_segdir'],  #'Segmentaton/',\n",
    "        s_format_segdir_cellpose = config.d_nconv['s_format_segdir_cellpose'],  #'{}{}_CellposeSegmentation/', # s_segdir, s_slide\n",
    "        s_qcdir = config.d_nconv['s_qcdir'],  #'QC/',\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ea757df-a347-4250-9b41-1d141e72adfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10.2 feature correct labels\n",
    "\n",
    "# it makes only sens to run this function, if s_thresh_marker is not None!\n",
    "\n",
    "# function expands cytoplasm negative cells from the matched nucleus cell label file by i_exp. \n",
    "# output in s_segdir/s_slide_CellposeSegmentation/:\n",
    "# {slide_scene}_{seg_marker}_nuc{cell_diam}_cell{nuc_diam}__matched_{exp}_CellSegmentationBasins.tif\n",
    "# celltouch_{slide}_{s_thresh_marker}_segmentation.json\n",
    "\n",
    "feat.feature_correct_labels_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    di_seg_marker = {'Ecad': 1000},  # {'Ecad': 1000} # choose same value as in 10.1 and in relation to s_afsubdir!\n",
    "    i_exp = 5,  # choose same value as in 10.1.\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_afsubdir = config.d_nconv['s_regdir'],  #'RegisteredImages/', or 'SubtractedRegisteredImages/',\n",
    "    s_segdir = config.d_nconv['s_segdir'],  #'Segmentation/',\n",
    "    s_format_segdir_cellpose = config.d_nconv['s_format_segdir_cellpose'],  #'{}{}_CellposeSegmentation/', # s_segdir, s_slide\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7480094-e176-4704-9c10-db0bfe42793c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fe74072-923b-4a43-adb6-7e1714db1671",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 11.0 qc plots for z projection, segmentation basin and tissue edge distance\n",
    "\n",
    "segment.nuccell_zprojlabel_imgs_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    es_seg_marker = {'Ecad'},  # choose same value as in 8.0.\n",
    "    s_tissue_dapi = 'DAPI1',  # choose same value as in 10.1.\n",
    "    i_tissue_dapi_thresh = 500,  # choose same value as in 10.1.\n",
    "    i_tissue_area_thresh = 50000,  # choose same value as in 10.1.\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_segdir = config.d_nconv['s_segdir'],  #'Segmentation/',\n",
    "    s_format_segdir_cellpose = config.d_nconv['s_format_segdir_cellpose'],  #'{}{}_CellposeSegmentation/',  # s_segdir, s_slide\n",
    "    s_qcdir = config.d_nconv['s_qcdir'],  #'QC/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c453142c-4d64-46ce-9d09-221892ee498f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6eda83f8-ede4-4174-8a5d-482f5d818133",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 12.0 work directory tidy up\n",
    "\n",
    "# the function generates a scripts folder and  puts all scripts that were run by the pipeline into it.\n",
    "# additionally, the function deletes all  slurm-*.out and slurp-*.out output files because they are no longer needed.\n",
    "\n",
    "util.sweep()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60cd71f0-ea95-4838-9447-8d47afa376a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ebf4f2f-5845-44a9-9f6b-a89cea6d6db9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ometiff  (optional)\n",
    "\n",
    "# function generates for each specifies slide_pxscene \n",
    "# a (non-pyramide) multichannel tiff with ome metadata (not totally complete).\n",
    "# + http://www.openmicroscopy.org/ome-files/\n",
    "\n",
    "ometiff.ometiff_spawn(\n",
    "    es_slide = ls_slide,  # slide filter\n",
    "    es_slide_pxscene = None,  # scene filter . If None, all scenes from the es_slide specified slides will be generated. \n",
    "    es_exclude_round = {'R0'},  # rounds to be excluded from the multichannel ometiff. \n",
    "    ddd_crop = ddd_crop,  # to map slide_mscene to slide_pxscene\n",
    "    ddd_etc = ddd_etc,  # exposure time correction\n",
    "    # microscopy\n",
    "    r_pixel_size_um = ,  # 0.325, # microscope dependent!\n",
    "    # experiment\n",
    "    s_batch_id = s_batch,  # experiment\n",
    "    s_lab = ,  # experimenter_group\n",
    "    s_email_leader = ,  # experimenter\n",
    "    # output image\n",
    "    b_8bit = False,  # if False, output will be like input tiffs, 16[bit].\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem ='64G',\n",
    "    s_slurm_time ='36:00:00',\n",
    "    s_slurm_account ='gray_lab',\n",
    "    # file system\n",
    "    s_afsubdir = config.d_nconv['s_regdir'],  #'SubtractedRegisteredImages/' 'RegisteredImages/',\n",
    "    s_format_afsubdir = config.d_nconv['s_format_regdir'],  #'{}{}/' # s_afsubdir, s_slide_pxscene,\n",
    "    s_metadir = config.d_nconv['s_metadir'],  #'MetaImages/',\n",
    "    s_ometiffdir = config.d_nconv['s_ometiffdir'],  #'OmeTiffImages/',\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "751a8d71-b0e8-417a-bd5d-f34b860c5ba3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be77f6a4-ac21-4b19-a4db-3339d9233524",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compress files (optional)\n",
    "\n",
    "# do save disk space, raw tiff and czi files can be compressed.\n",
    "# compression needs time and process power. \n",
    "# decompression is relative quick.\n",
    "\n",
    "util.compress_xz_spawn(\n",
    "    es_slide = ls_slide,\n",
    "    b_tiff_raw = False,  # compress raw tiff files?\n",
    "    b_czi_original = False,  # compress original czi files?    \n",
    "    b_czi_splitscene = False,  # compress splitscene czi files?\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  # 'RawImages/',\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/', # s_rawdir, s_slide\n",
    "    s_czidir = config.d_nconv['s_czidir'],  # 'CziImages/',\n",
    "    s_format_czidir_original = config.d_nconv['s_format_czidir_original'],  #'{}{}/original/', # s_czidir, s_slide\n",
    "    s_format_czidir_splitscene = config.d_nconv['s_format_czidir_splitscene'],  #'{}{}/splitscene/', # s_czidir, s_slide\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a291402-0f05-49f8-bb0b-ddb5eb7ba813",
   "metadata": {},
   "outputs": [],
   "source": [
    "# de-compress xz files raw tiff.xz and czi.xz files (do not run, if not necessary)\n",
    "\n",
    "# compression needs time and process power. \n",
    "# decompression is relative quick.\n",
    "\n",
    "util.decompress_xz_spawn(\n",
    "    es_slide = ls_slide,  # run this function only for the necessary slides!\n",
    "    b_tiff_raw = False,   # de-compress raw tiff files?\n",
    "    b_czi_original = False,  # de-compress original czi files?\n",
    "    b_czi_splitscene = False,  # de-compress splitscene czi files?\n",
    "    # processing\n",
    "    s_type_processing = 'slurm',\n",
    "    s_slurm_partition = 'exacloud',\n",
    "    s_slurm_mem = '64G',\n",
    "    s_slurm_time = '36:00:00',\n",
    "    s_slurm_account = 'gray_lab',\n",
    "    # file system\n",
    "    s_rawdir = config.d_nconv['s_rawdir'],  #'RawImages/',\n",
    "    s_format_rawdir = config.d_nconv['s_format_rawdir'],  #'{}{}/', # s_rawdir, s_slide\n",
    "    s_czidir = config.d_nconv['s_czidir'],  # 'CziImages/',\n",
    "    s_format_czidir_original = config.d_nconv['s_format_czidir_original'],  #'{}{}/original/', # s_czidir, s_slide\n",
    "    s_format_czidir_splitscene = config.d_nconv['s_format_czidir_splitscene'],  #'{}{}/splitscene/', # s_czidir, s_slide\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d78db73-9d61-49b8-90db-39b7d0c6a887",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mplexable",
   "language": "python",
   "name": "mplexable"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}