Efficiency Improvements + Protect Data Temp Directory

examples/fmri/biobank/2_train_hmm.py

@@ -35,7 +35,12 @@
     inputs = file.read().split("\n")
 
 # Create Data object for training
-data = Data(inputs, load_memmaps=False, n_jobs=8)
+data = Data(
+    inputs,
+    use_tfrecord=True,
+    store_dir=f"tmp_{id}",
+    n_jobs=8,
+)
 
 # Prepare data
 data.standardize()

examples/fmri/biobank/4_dual_estimation.py

@@ -47,7 +47,12 @@
     inputs = file.read().split("\n")
 
 # Create Data object for training
-data = Data(inputs, load_memmaps=False, n_jobs=8)
+data = Data(
+    inputs,
+    use_tfrecord=True,
+    store_dir=f"tmp_{id}",
+    n_jobs=16,
+)
 
 # Prepare data
 data.standardize()

examples/fmri/biobank/submit_jobs.py

@@ -4,7 +4,7 @@
 
 import os
 
-def write_job_script(run, queue="gpu_short"):
+def write_job_script(run, queue="short", n_gpus=1, n_cpus=12):
     """Create a job script to submit."""
 
     with open("job.sh", "w") as file:
@@ -14,12 +14,17 @@ def write_job_script(run, queue="gpu_short"):
         file.write(f"#SBATCH -o logs/{name}.out\n")
         file.write(f"#SBATCH -e logs/{name}.err\n")
         file.write(f"#SBATCH -p {queue}\n")
-        file.write("#SBATCH --gres gpu:1\n")
+        if "gpu" in queue:
+            file.write(f"#SBATCH --gres gpu:{n_gpus}\n")
+        else:
+            file.write(f"#SBATCH -c {n_cpus}\n")
         file.write("source activate osld\n")
         file.write(f"python 2_train_hmm.py {run}\n")
 
+# Create directory to hold log/error files
 os.makedirs("logs", exist_ok=True)
 
+# Submit jobs
 for run in range(1, 11):
     write_job_script(run)
     os.system("sbatch job.sh")

examples/fmri/hcp/2_train_hmm.py

@@ -35,7 +35,11 @@
     inputs = file.read().split("\n")
 
 # Create Data object for training
-data = Data(inputs, load_memmaps=False, n_jobs=8)
+data = Data(
+    inputs,
+    store_dir=f"tmp_{id}",
+    n_jobs=8,
+)
 
 # Prepare data
 data.standardize()

examples/fmri/hcp/4_dual_estimation.py

@@ -47,7 +47,11 @@
     inputs = file.read().split("\n")
 
 # Create Data object for training
-data = Data(inputs, load_memmaps=False, n_jobs=8)
+data = Data(
+    inputs,
+    store_dir=f"tmp_{id}",
+    n_jobs=8,
+)
 
 # Prepare data
 data.standardize()

examples/fmri/hcp/submit_jobs.py

@@ -4,7 +4,7 @@
 
 import os
 
-def write_job_script(run, queue="gpu_short"):
+def write_job_script(run, queue="short", n_gpus=1, n_cpus=12):
     """Create a job script to submit."""
 
     with open("job.sh", "w") as file:
@@ -14,12 +14,17 @@ def write_job_script(run, queue="gpu_short"):
         file.write(f"#SBATCH -o logs/{name}.out\n")
         file.write(f"#SBATCH -e logs/{name}.err\n")
         file.write(f"#SBATCH -p {queue}\n")
-        file.write("#SBATCH --gres gpu:1\n")
+        if "gpu" in queue:
+            file.write(f"#SBATCH --gres gpu:{n_gpus}\n")
+        else:
+            file.write(f"#SBATCH -c {n_cpus}\n")
         file.write("source activate osld\n")
         file.write(f"python 2_train_hmm.py {run}\n")
 
+# Create directory to hold log/error files
 os.makedirs("logs", exist_ok=True)
 
+# Submit jobs
 for run in range(1, 11):
     write_job_script(run)
     os.system("sbatch job.sh")

osl_dynamics/data/__init__.py

@@ -10,6 +10,7 @@
   /tutorials_build/data_preparation.html>`_
 """
 
-from osl_dynamics.data.base import Data, SessionLabels, load_tfrecord_dataset
+from osl_dynamics.data.base import Data, SessionLabels
+from osl_dynamics.data.tf import load_tfrecord_dataset
 
-__all__ = ["Data"]
+__all__ = ["Data", "SessionLabels", "load_tfrecord_dataset"]