forked from Deep-MI/FastSurfer
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
26d4e2b
commit 0312b98
Showing
3 changed files
with
175 additions
and
29 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,148 @@ | ||
| import logging | ||
| import os | ||
| import tarfile | ||
|
|
||
| from argparse import Namespace | ||
| from dataclasses import dataclass | ||
| from pathlib import Path | ||
| from utils import logTitle | ||
|
|
||
| # # Silence obnoxious messages from nda-tools | ||
| # with suppress_stdout(): | ||
| from NDATools import prerun_checks_and_setup | ||
| from NDATools.Download import Download | ||
| from NDATools.Configuration import ClientConfiguration | ||
|
|
||
| ''' | ||
| This module provides programmatic access to the 'download' methods | ||
| available as part of NDA-tools. It first (createArgs) creates the argument | ||
| parser namespace that would normally be created from command line arguments. | ||
| We'll pass the raw subject_ses string to the regex filter argument to download | ||
| only the files for the current subject/timepoint pairing. It then checks for a | ||
| default NDA-tools configuration file in HOME/.NDATools and attempts to create a | ||
| new file (and parent directory) if it doesn't find one. It then starts the | ||
| downloading the TGZ archives containing each file. After the download is complete, | ||
| it decompresses the archives and returns ONLY the files, ignoring any internal | ||
| directory structure. | ||
|
|
||
| ###IMPORTANT#### | ||
| This module relies on the python 'keyring' and 'keyrings.alt' modules (yes, they're distinct) | ||
| being installed, as they'll be used by NDATools. Keyring manages the access credentials | ||
| to NDA while keyrings provides alternate formats for those credentials. Currently, | ||
| we're using the less than optimal DCAN-Labs approach of using plain text storage | ||
| for our keyring - an option made possible by keyrings - which MUST BE CONFIGURED MANUALLY | ||
| (unless I get around to using something better). Specifically, create a file | ||
| named 'keyringrc.cfg' in the directory $HOME/.config/python_keyring . Add the | ||
| following text inside it... | ||
|
|
||
| [backend] | ||
| default-keyring=keyrings.alt.file.PlaintextKeyring | ||
| keyring-path=/tmp/work | ||
|
|
||
| ...to enable plain text keyrings, which will be stored in /tmp/work. Afterwards, set up | ||
| keyring authentication by loading a python terminal and running: | ||
|
|
||
| import keyring | ||
| keyring.set_password("nda-tools", "username", "password") | ||
|
|
||
| ...where "username" and "password" are the credentials TO THE NDA WEBSITE (NOT the miNDAR) | ||
| ############################### | ||
| ''' | ||
|
|
||
| @dataclass | ||
| class Downloader: | ||
| rawsubjses_str: str | ||
| input_dir: str | ||
| NDA_username: str | ||
| miNDAR_packageID: str | ||
| nThreads: str | ||
|
|
||
| def __post_init__(self): | ||
| self.logger = logging.getLogger('Downloading') | ||
| logTitle(f'Downloading and Unpacking Scan Files for {self.rawsubjses_str}') | ||
| self.main() | ||
|
|
||
| def createArgs(self): | ||
| # nda-tools makes us pass an 'args' namespace to the download module | ||
| # that contains the following contents | ||
| args = Namespace( | ||
| package=self.miNDAR_packageID, | ||
| file_regex=self.rawsubjses_str, | ||
| username=self.NDA_username, | ||
| directory=[str(self.input_dir)], | ||
| txt="", | ||
| datastructure="", | ||
| s3_destination="", | ||
| paths="", | ||
| workerThreads=self.nThreads, | ||
| verify=False) | ||
| return (args) | ||
|
|
||
|
|
||
| # Check if an NDATools configuration settings file exists, and create one if it does not | ||
| # Load the configuration settings into 'config' | ||
| def configCheck(self): | ||
| defaultConfigFile = Path.home() / ".NDATools/settings.cfg" | ||
| if defaultConfigFile.is_file(): | ||
| config = ClientConfiguration( | ||
| self.NDA_username | ||
| ) | ||
| config._check_and_fix_missing_options() | ||
| config.read_user_credentials() | ||
| else: | ||
| config = ClientConfiguration( | ||
| "clientscripts/config/settings.cfg", self.NDA_username) | ||
| config.read_user_credentials() | ||
|
|
||
| return config | ||
|
|
||
| # Initiate the download for this subject_ses | ||
| def download(self, config, args): | ||
|
|
||
| rawsubj_str = self.rawsubjses_str.split("_")[0] | ||
| rawses_str = self.rawsubjses_str.split("_")[1] | ||
|
|
||
| try: | ||
| self.logger.info(f'Now downloading scan files for subject ' | ||
| f'{rawsubj_str}, timepoint {rawses_str}...') | ||
| s3Download = Download(config, args) | ||
| s3Download.start() | ||
|
|
||
| except Exception as exc: | ||
| print(exc) | ||
| self.logger.exception(exc, exc_info=True) | ||
|
|
||
| # Loop over all the tgz files in the subject_ses folder and unpack them | ||
| def unpack(self): | ||
| self.logger.info('Unpacking all the TGZ files...') | ||
|
|
||
| def niiextract(tgz): | ||
| try: | ||
| with tarfile.open(tgz, "r") as tar: | ||
| # Only extract the NIFTI files | ||
| members = [member for member in tar.getmembers() if member.name.split(".")[-1] == "nii"] | ||
| for member in members: | ||
| # Don't extract parent folders, just the files | ||
| member.name = Path(member.name).name | ||
| self.logger.info(f'Decompressing {member.name}...') | ||
| tar.extractall(path=self.input_dir, members=members) | ||
| except Exception as exc: | ||
| self.logger.exception(f'Failed to unpack {tgz}: {exc}', exc_info=True) | ||
| else: | ||
| os.remove(tgz) | ||
|
|
||
| [niiextract(tgz) for tgz in Path(self.input_dir).rglob(f'{self.rawsubjses_str}*.tgz')] | ||
|
|
||
| def main(self): | ||
| try: | ||
| self.logger = logging.getLogger(__name__) | ||
| prerun_checks_and_setup() | ||
| args = self.createArgs() | ||
| config = self.configCheck() | ||
| self.download(config, args) | ||
| self.unpack() | ||
| except Exception as exc: | ||
| self.logger.exception(f'Encountered exception while downloading and unpacking files: {exc}', exc_info=True) | ||
|
|
||
|
|
||
| Downloader('NDARINV00LH735Y_baselineYear1Arm1', "/fastscratch/jdr/ABCD/fastSurf/inputs", "jrusse10", "1226193", "2") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters