Skip to content

Commit

Permalink
Merge pull request #35 from jjjermiah/24-nbiapyhandle-error-case-of-i…
Browse files Browse the repository at this point in the history 
…f-resposne-is-not-bytes-for-downloadseries

fix(nbia): add error handling for downloadSeries
  • Loading branch information
@jjjermiah
jjjermiah committed Jan 7, 2024
2 parents b2e56fe + e6cc91e commit 1f0a499
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 96 deletions.
5 changes: 4 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,4 +145,7 @@ olddicomsort.py
NBIA-toolkit.code-workspace
docs/data/*
driver.py
data/*
data/*
sandbox

docs/data/*
176 changes: 98 additions & 78 deletions docs/Tutorial.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,13 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 14:13 | NBIAClient | DEBUG | Setting up OAuth2 client... with username nbia_guest\n"
]
}
],
"outputs": [],
"source": [
"from nbiatoolkit import NBIAClient\n",
"from pprint import pprint\n",
"\n",
"# Instantiate the client. \n",
"client = NBIAClient(log_level='debug')\n",
"client = NBIAClient(log_level='info')\n",
"\n",
"# NOTE::READTHEDOCS \n",
"# THIS WILL FAIL UNTIL WE UPLOAD PACKAGE TO PYPI AND INSTALL VIA PIP"
Expand Down Expand Up @@ -74,25 +66,18 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 14:13 | NBIAClient | DEBUG | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getCollectionValues\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['4D-Lung',\n",
" 'A091105',\n",
" 'ACNS0332',\n",
" 'ACRIN-6698',\n",
" 'ACRIN-Contralateral-Breast-MR']\n"
" 'ACRIN-Contralateral-Breast-MR',\n",
" 'ACRIN-FLT-Breast',\n",
" 'ACRIN-NSCLC-FDG-PET']\n"
]
}
],
Expand All @@ -111,25 +96,18 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 13:08 | NBIAClient | DEBUG | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/getCollectionValuesAndCounts\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[{'Collection': '4D-Lung', 'PatientCount': '20'},\n",
" {'Collection': 'A091105', 'PatientCount': '83'},\n",
" {'Collection': 'ACNS0332', 'PatientCount': '85'},\n",
" {'Collection': 'ACRIN-6698', 'PatientCount': '385'},\n",
" {'Collection': 'ACRIN-Contralateral-Breast-MR', 'PatientCount': '984'}]\n"
" {'Collection': 'ACRIN-Contralateral-Breast-MR', 'PatientCount': '984'},\n",
" {'Collection': 'ACRIN-FLT-Breast', 'PatientCount': '83'},\n",
" {'Collection': 'ACRIN-NSCLC-FDG-PET', 'PatientCount': '242'}]\n"
]
}
],
Expand All @@ -148,27 +126,20 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 13:08 | NBIAClient | DEBUG | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/getBodyPartValuesAndCounts\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Total Number of Collections:98\n",
"Total Number of Collections:60\n",
"First 5 body parts:\n",
"[{'BodyPartExamined': 'NOT SPECIFIED', 'Count': '8490'},\n",
" {'BodyPartExamined': 'ABDOMEN', 'Count': '2968'},\n",
"[{'BodyPartExamined': 'NOT SPECIFIED', 'Count': '7839'},\n",
" {'BodyPartExamined': 'ABDOMEN', 'Count': '1731'},\n",
" {'BodyPartExamined': 'ABDOMEN CAVIT', 'Count': '2'},\n",
" {'BodyPartExamined': 'ABDOMENPELVIC', 'Count': '2'},\n",
" {'BodyPartExamined': 'ABDOMENPELVIS', 'Count': '51'}]\n"
" {'BodyPartExamined': 'ABDOMENPELVIS', 'Count': '50'}]\n"
]
}
],
Expand Down Expand Up @@ -197,17 +168,9 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 13:28 | NBIAClient | DEBUG | Parsing params: {'self': <nbiatoolkit.nbia.NBIAClient object at 0x11100eba0>, 'Collection': '4D-Lung', 'Modality': 'CT'}\n",
"23-12-02 13:28 | NBIAClient | DEBUG | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getPatientByCollectionAndModality\n"
]
},
{
"name": "stdout",
"output_type": "stream",
Expand Down Expand Up @@ -249,16 +212,9 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"23-12-02 13:34 | NBIAClient | DEBUG | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getSeries\n"
]
},
{
"name": "stdout",
"output_type": "stream",
Expand Down Expand Up @@ -294,7 +250,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 8,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -337,14 +293,17 @@
"# download Series Data\n",
"``` python\n",
"downloadSeries(\n",
" SeriesInstanceUID: str,\n",
" downloadDir: str)\n",
" SeriesInstanceUID: Union[str, list],\n",
" downloadDir: str,\n",
" filePattern: str,\n",
" overwrite: bool,\n",
" nParallel: int)\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"outputs": [
{
Expand All @@ -357,7 +316,7 @@
" '1.3.6.1.4.1.14519.5.2.1.6834.5010.139116724721865252687455544825']"
]
},
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -370,38 +329,42 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"2023-11-20 16:55:28,455 | NBIAClient | INFO | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getImageWithMD5Hash\n"
"Downloading to: /Users/bhklab/Documents/GitHub/NBIA-toolkit/docs/data\n"
]
},
{
"name": "stdout",
"name": "stderr",
"output_type": "stream",
"text": [
"Downloading to: /home/bioinf/bhklab/jermiah/projects/NBIA-toolkit/docs/data\n"
"Downloading 5 series: 0%| | 0/5 [00:00<?, ?it/s]"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2023-11-20 16:55:33,722 | NBIAClient | INFO | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getImageWithMD5Hash\n",
"2023-11-20 16:55:39,813 | NBIAClient | INFO | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getImageWithMD5Hash\n",
"2023-11-20 16:55:44,858 | NBIAClient | INFO | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getImageWithMD5Hash\n",
"2023-11-20 16:55:49,399 | NBIAClient | INFO | Querying API endpoint: https://services.cancerimagingarchive.net/nbia-api/services/v2/getImageWithMD5Hash\n"
"Downloading 5 series: 100%|██████████| 5/5 [00:10<00:00, 2.10s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['1.3.6.1.4.1.14519.5.2.1.6834.5010.227929163446067537882961857921', '1.3.6.1.4.1.14519.5.2.1.6834.5010.336250251691987239290048605884', '1.3.6.1.4.1.14519.5.2.1.6834.5010.189721824525842725510380467695', '1.3.6.1.4.1.14519.5.2.1.6834.5010.925990093742075237571072608963', '1.3.6.1.4.1.14519.5.2.1.6834.5010.139116724721865252687455544825']\n"
"['P100']\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
Expand All @@ -411,13 +374,70 @@
"downloadDir = \"./data\"\n",
"os.makedirs(downloadDir, exist_ok=True)\n",
"print(\"Downloading to: \" + os.path.abspath(downloadDir))\n",
"for seriesUID in seriesUIDS[0:5]:\n",
" client.downloadSeries(seriesUID, downloadDir)\n",
"\n",
"cores = 4 # number of parallel downloads\n",
"client.downloadSeries(\n",
" seriesUIDS[0:5], downloadDir, overwrite=True, nParallel=cores)\n",
" \n",
"pprint(os.listdir(downloadDir))\n",
" "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Configure File names during download\n",
"\n",
"Due to the unique nature of the data in NBIA, the file names are not always consistent.\n",
"\n",
"To configure the file names during download you can pass in a parameter called `filePattern` to the `downloadSeries` method which is used by the `DICOMSorter`. For more information on how to configure the `filePattern` see the `nbiatoolkit.DICOMSorter()` class.\n",
"\n",
"The filePattern is a string of DICOM tags indicated by a `%` that are extracted from each DICOM file metadata and used to create the file name: \n",
"- i.e `%PatientName%_%SeriesInstanceUID%.dcm` will create a file name with the PatientName and SeriesInstanceUID.\n",
" - note: the UIDs will be shortened to the final 5 characters to avoid long file names.\n",
"\n",
"The default filePattern is : `%PatientName/%StudyDescription-%StudyDate/%SeriesNumber-%SeriesDescription-%SeriesInstanceUID/%InstanceNumber.dcm`. This will create the following tree structure:\n",
"\n",
"``` json\n",
"PatientName\n",
"└── StudyDescription-StudyDate\n",
"    └── SeriesNumber-SeriesDescription-SeriesInstanceUID\n",
"    └── InstanceNumber.dcm\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Downloading 5 series: 100%|██████████| 5/5 [00:05<00:00, 1.13s/it]\n"
]
},
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.downloadSeries(\n",
" seriesUIDS[0:5], \n",
" downloadDir, \n",
" filePattern=\"%PatientName/%SeriesNumber-%SeriesInstanceUID/%InstanceNumber-%SOPInstanceUID.dcm\",\n",
" overwrite=True, nParallel=4)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down
22 changes: 21 additions & 1 deletion poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ readme = "README.md"
python = "^3.12"
requests = "2.31.0"
pydicom = "^2.4.3"
tqdm = "^4.66.1"

[tool.poetry.dev-dependencies]

Expand Down

0 comments on commit 1f0a499

Please sign in to comment.