merge: Merge branch 'development' of https://github.com/jjjermiah/NBI…

…A-toolkit into development
jjjermiah · Feb 17, 2024 · 8c9470b · 8c9470b
2 parents 8c2367f + e71eac1
commit 8c9470b
Show file tree

Hide file tree

Showing 43 changed files with 10,584 additions and 1,171 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,21 +1,21 @@
-name: Unit Tests
-  
+name: CI-CD
+
 # only run on pushes to main or pull requests
 on:
   push:
-    # push to any branch
-    branches: [ main ]
+    # push to any branch *
+    branches: [ "*"]
   pull_request:
     branches: [ main , development]
-  
-jobs:  
-  Continuous-Integration:
+
+jobs:
+  Unit-Tests:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, macos-14]
         python-version: ["3.12", "3.11"] # "3.10",
-  
+
     steps:
       - uses: actions/checkout@v3
 
@@ -32,19 +32,19 @@ jobs:
 
       - name: Test with pytest
         run: |
-          poetry run pytest -n 2 --cov --cov-report xml:coverage-report/coverage.xml
-
-      - name: Build documentation
-        run: poetry run make html --directory docs/
+          # Github action runners now have 4 cores
+          poetry run pytest -n 4 --cov --cov-report xml:coverage-report/coverage.xml
 
-      - name: Upload coverage report
+      - name: Upload coverage report artifact to be used by Codecov
+        # only upload if matrix.os is ubuntu-latest and matrix.python-version is 3.12
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
         uses: actions/upload-artifact@v2
         with:
           name: coverage-report
           path: coverage-report
 
   Codecov:
-    needs: Continuous-Integration
+    needs: Unit-Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -65,7 +65,29 @@ jobs:
           verbose: true
           name: codecov-umbrella
 
+  Build-Documentation:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/docs'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.4
+
+      - name: Install poetry
+        uses: snok/install-poetry@v1
+
+      - name: Install dependencies
+        run: poetry install
 
+      - name: Build documentation
+        run: |
+          pip install -r docs/requirements.txt
+          pip install .
+          make html --directory=docs
 
   Continuous-Deployment:
     permissions:
@@ -74,10 +96,10 @@ jobs:
         issues: write
         pull-requests: write
 
-    # needs: Continuous-Integration
+    needs: [Unit-Tests]
 
     # if pulling to main, deploy to PyPI
-    # if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main'
 
     # Set up operating system
     runs-on: ubuntu-latest
@@ -100,11 +122,6 @@ jobs:
     - name: Install package
       run: poetry install
 
-
-    # Existing jobs...
-
-
-
    # This action uses Python Semantic Release v8
    # What this action does:
    # - Determines the next version number based on the commit history
@@ -116,7 +133,6 @@ jobs:
       uses: python-semantic-release/python-semantic-release@master
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        git_commit_message: 'chore(release): release ${next_version}'
 
     - name: Install packaging-related tool
       run:
@@ -142,16 +158,23 @@ jobs:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
 
-    - name: Print latest tag name
-      run: |
-        printf "%s\n" "${LATEST_TAG}"
-
     - name: Print branch name
       run: |
         printf "LATEST TAG: %s\n" "${LATEST_TAG}"
         echo "DOCKER_IMAGE_TAG=${LATEST_TAG}" >> "$GITHUB_ENV"
         printf "DOCKER_IMAGE_TAG: %s\n" "${DOCKER_IMAGE_TAG}"
 
+    - name: Print outputs from semantic release
+      run: |
+        # Semantic release gha provides the following outputs:
+        # - released: true if a new release was created, false otherwise
+        # - version: the new version number
+        # - tag: The Git tag that was created
+        echo "${{ steps.release.outputs.released }}"
+        echo "${{ steps.release.outputs.version }}"
+        echo "${{ steps.release.outputs.tag }}"
+        echo "${{ steps.release.outputs.release_notes }}"
+
     - name: Set up QEMU
       if: steps.release.outputs.released == 'true'
       uses: docker/setup-qemu-action@v3
@@ -205,15 +228,15 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, macos-14]
         python-version: ["3.12", "3.11"]
-    
+
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Install using PyPi
         run: |
           pip install  nbiatoolkit;
@@ -321,7 +344,7 @@ jobs:
               git add .
 
               # Commit with a timestamp
-              git commit -m "Update README with latest version: $LATEST_TAG"
+              git commit -m "chore: Update README: $LATEST_TAG"
 
               # Push changes to the remote repository
               # if github.head_ref is not null
@@ -342,10 +365,10 @@ jobs:
 
 
 
-          # git add README.md
+  #         # git add README.md
 
-          # git commit -m "Update README with latest version"
-          # git push
+  #         # git commit -m "Update README with latest version"
+  #         # git push
 
 
 # #!/bin/bash
@@ -369,4 +392,3 @@ jobs:
 #     # No changes
 #     echo "No changes found. Nothing to commit or push."
 # fi
-
diff --git a/.gitignore b/.gitignore
@@ -148,4 +148,10 @@ driver.py
 data/*
 sandbox
 
-docs/data/*
+docs/data/*
+docs/autoapi
+src/nbiatoolkit/.DS_Store
+
+NBIA-Download
+src/nbiatoolkit/.DS_Store
+.vscode
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,19 @@
+repos:
+-   repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: check-json
+    -   id: check-toml
+
+-   repo: https://gitlab.com/smop/pre-commit-hooks
+    rev: v1.0.0
+    hooks:
+    -   id: check-poetry
diff --git a/Dockerfile b/Dockerfile
@@ -4,19 +4,24 @@
 # To build the container, run the following command from the root of the
 # repository:
 # docker build -t nbiatoolkit .
-#
-# ADD LABELS HERE
+
+
 FROM python:3.12-slim
 
-LABEL maintainer="Jermiah Joseph"
+LABEL maintainer="Jermiah Joseph jermiahjoseph98@gmail.com"
+LABEL description="This is a Dockerfile for the nbiatoolkit package."
+LABEL license="MIT"
+LABEL usage="docker run -it --rm <image_name> NBIAToolkit --help"
+LABEL org.opencontainers.image.source="github.com/jjjermiah/nbiatoolkit"
 
 # install the dependencies
+RUN pip install --upgrade pip
 RUN python -m pip install nbiatoolkit
 
-RUN python -c 'import nbiatoolkit; print(nbiatoolkit.version())'
+RUN NBIAToolkit --help
 
 # On run, open a bash shell
 CMD ["/bin/bash"]
 
 # to run this container in terminal mode, use the following command:
-# docker run -it --rm nbiatoolkit
+# docker run -it --rm nbiatoolkit
diff --git a/LICENSE b/LICENSE
@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-
diff --git a/README.md b/README.md
@@ -1,4 +1,5 @@
 # *NBIA Toolkit*
+## A python package that provides programmatic access to query and download images from the National Biomedical Imaging Archive (**NBIA**) and The Cancer Imaging Archive (**TCIA**) databases.
 [![PyTests](https://github.com/jjjermiah/nbia-toolkit/actions/workflows/main.yml/badge.svg)](https://github.com/jjjermiah/nbia-toolkit/actions/workflows/main.yml)
 [![Documentation Status](https://readthedocs.org/projects/nbia-toolkit/badge/?version=latest)](https://nbia-toolkit.readthedocs.io/en/latest/?badge=latest)
 [![codecov](https://codecov.io/gh/jjjermiah/nbia-toolkit/graph/badge.svg?token=JKREY71D0R)](https://codecov.io/gh/jjjermiah/nbia-toolkit)
@@ -9,69 +10,88 @@
 [![PyPI - Downloads](https://img.shields.io/pypi/dm/nbiatoolkit.svg?label=pypi%20downloads)](https://pypi.org/project/nbiatoolkit/)
 ![GitHub repo size](https://img.shields.io/github/repo-size/jjjermiah/nbia-toolkit)
 [![Docker Pulls](https://img.shields.io/docker/pulls/jjjermiah/nbiatoolkit)](https://hub.docker.com/r/jjjermiah/nbiatoolkit)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
 
 
 
 ![GitHub milestone details](https://img.shields.io/github/milestones/progress-percent/jjjermiah/nbia-toolkit/1?style=flat-square&label=1.0.0%20Stable%20Release%20Milestone&link=https%3A%2F%2Fgithub.com%2Fjjjermiah%2Fnbia-toolkit%2Fmilestone%2F1)![GitHub milestone details](https://img.shields.io/github/milestones/progress/jjjermiah/nbia-toolkit/1?style=flat-square&label=%20&link=https%3A%2F%2Fgithub.com%2Fjjjermiah%2Fnbia-toolkit%2Fmilestone%2F1)
 
 
 
-`nbiatoolkit` is a python package that provides programmatic access to query and download images from the National Biomedical Imaging Archive (**NBIA**) and The Cancer Imaging Archive (**TCIA**) databases.
-
 ## Features
-- Use NBIA Guest account to access public data OR authenticate using OAuth with user credentials for limited access data (requires approved data access).
+> [!TIP]
+> For a thorough description of the package and its available features, please refer to the Documentation at [NBIA-Toolkit Read The Docs](https://nbia-toolkit.readthedocs.io/en/latest/)
+
+- ***Programmatic access*** to the National Biomedical Imaging Archive (NBIA) and The Cancer Imaging Archive (TCIA) databases
+  - Use NBIA Guest account to access public data OR authenticate using OAuth with user credentials for limited access data (requires approved data access).
+  - Custom `OAuth2` class for **NBIA**, **TCIA**, including special handling for dedicated server for the **NLST** collection.
+
 - ***Query NBIA database*** for metadata on ***collections***, ***patients***, ***studies***, ***series***, and ***images***
 - Download images from NBIA
   - ***Validate doownloads with MD5 checksums*** for downloaded images
   - **Auto-sort** DICOM files using a user-defined pattern of DICOM tags with specialized ***DICOMSorter class***
 
-See Documentation at [NBIA-Toolkit Read The Docs](https://nbia-toolkit.readthedocs.io/en/latest/)
 
 
 ## Installation
 
-`nbiatoolkit` is currently under development.
+> [!WARNING]
+> `nbiatoolkit` is currently under development and is not guaranteed to be stable.
+
 It is made available via PyPI and can be installed using pip:
 ****
 ```bash
 pip install nbiatoolkit
 ```
 
 ## CLI Usage
+For quick access to the NBIA, the toolkit also provides a command line interface (CLI)
 
-# getCollections Output
-nbia-toolkit also provides a command line interface (CLI) to query the NBIA database for some queries.
 ``` bash NBIAToolkit-Output
 > NBIAToolkit --version
-    _   ______  _______  ______            ____   _ __ 
+    _   ______  _______  ______            ____   _ __
    / | / / __ )/  _/   |/_  __/___  ____  / / /__(_) /_
   /  |/ / __  |/ // /| | / / / __ \/ __ \/ / //_/ / __/
- / /|  / /_/ // // ___ |/ / / /_/ / /_/ / / ,< / / /_  
-/_/ |_/_____/___/_/  |_/_/  \____/\____/_/_/|_/_/\__/  
-
+ / /|  / /_/ // // ___ |/ / / /_/ / /_/ / / ,< / / /_
+/_/ |_/_____/___/_/  |_/_/  \____/\____/_/_/|_/_/\__/
+
+
+Version: 0.29.2
 
-Version: 0.19.2
+Available CLI tools:
 
-Available CLI tools: 
+getCollections [-h] [-u USERNAME] [-pw PASSWORD] [-p PREFIX]
+               [-o OUTPUTFILE] [--version]
 
-getCollections [-h] [-p PREFIX] [-o OUTPUTFILE] [--version]
+getBodyPartCounts [-h] [-u USERNAME] [-pw PASSWORD] [-c COLLECTION]
+                  [-o OUTPUTFILE] [--version]
 
-getPatients [-h] -c COLLECTION [-o OUTPUTFILE] [--version]
+getPatients [-h] [-u USERNAME] [-pw PASSWORD] -c COLLECTION
+            [-o OUTPUTFILE] [--version]
 
-getBodyPartCounts [-h] [-c COLLECTION] [-o OUTPUTFILE] [--version]
+getNewPatients [-h] [-u USERNAME] [-pw PASSWORD] -c COLLECTION -d DATE
+               [-o OUTPUTFILE] [--version]
 
-getSeries [-h] [-c COLLECTION] [-p PATIENTID] [-m MODALITY]
-          [-study STUDYINSTANCEUID]
+getStudies [-h] [-u USERNAME] [-pw PASSWORD] -c COLLECTION
+           [-p PATIENTID] [-s STUDYINSTANCEUID] [-o OUTPUTFILE]
+           [--version]
+
+getSeries [-h] [-u USERNAME] [-pw PASSWORD] [-c COLLECTION]
+          [-p PATIENTID] [-m MODALITY] [-study STUDYINSTANCEUID]
           [--seriesInstanceUID SERIESINSTANCEUID]
           [--bodyPartExamined BODYPARTEXAMINED]
           [--manufacturerModelName MANUFACTURERMODELNAME]
           [--manufacturer MANUFACTURER] [-o OUTPUTFILE] [--version]
 
-downloadSingleSeries [-h] --seriesUID SERIESUID --downloadDir
-                     DOWNLOADDIR [--filePattern FILEPATTERN]
-                     [--overwrite]
+getNewSeries [-h] [-u USERNAME] [-pw PASSWORD] -d DATE [-o OUTPUTFILE]
+             [--version]
+
+downloadSingleSeries [-h] [-u USERNAME] [-pw PASSWORD] --seriesUID
+                     SERIESUID --downloadDir DOWNLOADDIR
+                     [--filePattern FILEPATTERN] [--overwrite]
 
-dicomsort [-h] [--targetPattern TARGETPATTERN] [--truncateUID]
+dicomsort [-h] [-u USERNAME] [-pw PASSWORD]
+          [--targetPattern TARGETPATTERN] [--truncateUID]
           [--sanitizeFilename] [--overwrite] [--nParallel NPARALLEL]
           sourceDir destinationDir
 
@@ -83,14 +103,11 @@ dicomsort [-h] [--targetPattern TARGETPATTERN] [--truncateUID]
 Interested in contributing? Check out the contributing guidelines. Please note that this project is released with a Code of Conduct. By contributing to this project, you agree to abide by its terms.
 
 ## License
-
 `nbiatoolkit` was created by Jermiah Joseph. It is licensed under the terms of the MIT license.
 
 ## User Agreements and Disclaimers
-The NBIA-toolkit is NOT a product of the National Cancer Institute (NCI) and is not endorsed by the NCI.
-The NBIA-toolkit is provided as an open-source tool based on the [NBIA REST API](https://wiki.cancerimagingarchive.net/display/Public/NBIA+Advanced+REST+API+Guide).
-The NBIA-toolkit is provided "AS IS" without warranty of any kind.
-
-In no event shall the authors or contributors be liable for any claim, damages or other liability, arising from, out of or in connection with the NBIA-toolkit or the use or other dealings in the NBIA-toolkit.
-
-Users of the NBIA-toolkit are required to abide by the NBIA REST API Terms of Service and the [NBIA Data Usage Policies and Restrictions](https://www.cancerimagingarchive.net/data-usage-policies-and-restrictions/)
+> [!IMPORTANT]
+>The NBIA-toolkit is NOT a product of the National Cancer Institute (NCI) and is not endorsed by the NCI.
+> Users of the NBIA-toolkit are required to abide by the NBIA REST API Terms of Service and the [NBIA Data Usage Policies and Restrictions](https://www.cancerimagingarchive.net/data-usage-policies-and-restrictions/)
+> The NBIA-toolkit is provided as an open-source tool based on the [NBIA REST API](https://wiki.cancerimagingarchive.net/display/Public/NBIA+Advanced+REST+API+Guide) and is provided "AS IS" without warranty of any kind.
+> In no event shall the authors or contributors be liable for any claim, damages or other liability, arising from, out of or in connection with the NBIA-toolkit or the use or other dealings in the NBIA-toolkit.