Merge pull request #153 from berkeley-stat159/Tests

Merging the Master and Tests branch - Tests branch CAN pass Travis for 2 tests

.gitignore

@@ -60,6 +60,11 @@ doc/source/_static/test.png
 data/ds005
 data/ds005_raw_checksums.txt
 data/ds005_mnifunc.tar
-data/ds005_2
+data/ds005_raw_checksums.txt
+data/ds005_filtered/
+data/dwn_filtered_data_script.sh
+data/ds114/
+data/ds107/
+
 \#*
 *~

.travis.yml

@@ -24,8 +24,6 @@ install:
     pip install coveralls;
     fi
 
-sudo: required
-
 script:
   - make coverage
 

Makefile

@@ -1,15 +1,15 @@
-.PHONY: all clean coverage test
+.PHONY: all clean coverage test verbose
 
 all: clean
 
 clean:
 	find . -name "*.so" -o -name "*.pyc" -o -name "*.pyx.md5" | xargs rm -f
 
 coverage:
-	nosetests code/utils data --with-coverage --cover-package=data  --cover-package=utils
+	nosetests code/utils/tests data/tests/ --with-coverage --cover-package=code/utils/functions,data/tests/test_get_check_hashes.py
 
 test:
-	nosetests code/utils data
+	nosetests code/utils/tests data/tests/
 
 verbose:
-	nosetests -v code/utils data
+	nosetests data/tests code/utils/tests -v

code/Makefile

@@ -1,5 +1,7 @@
+.PHONY: test coverage
+
 test:
-	nosetests utils
+	nosetests utils/tests -w ..
 
 coverage:
-	nosetests utils --with-coverage --cover-package=utils 
+	nosetests utils/tests -w --with-coverage --cover-package=utils/functions 

code/utils/tests/test_pearson_1d.py

@@ -6,32 +6,17 @@
     nosetests test_pearson_1d.py
 
 This is a test module.
-
-It is designed to be run the with the "nose" testing package (via the
-"nosetests" script.
-
-Nose will look for any functions with "test" in their names, and run them.
-Nose reports any errors, or any failures.
-
-A failure is where one of the test conditions run with an "assert" command
-fails.  For example, if I did:
-
-    assert_almost_equal(1, 2)
-
-then this would "fail".
-
-So we use the tests to check that the results of our function are (still) as we
-expect.
 """
 # Python 3 compatibility
 from __future__ import absolute_import, division, print_function
+from numpy.testing import assert_almost_equal
 
 import numpy as np
+import sys, os, pdb
 
-from .. import pearson
-
-from numpy.testing import assert_almost_equal
-
+#Specicy the path for functions
+sys.path.append(os.path.join(os.path.dirname(__file__), "../functions/"))
+import pearson
 
 def test_pearson_1d():
     # Test pearson_1d routine

code/utils/tests/test_pearson_2d.py

@@ -4,16 +4,18 @@
 Run with::
 
     nosetests test_pearson_2d.py
+This is a test module.
 """
 # Python 3 compatibility
 from __future__ import print_function, division
+from numpy.testing import assert_almost_equal
 
 import numpy as np
+import sys, os, pdb
 
-from .. import pearson
-
-from numpy.testing import assert_almost_equal
-
+#Specify the path for functions
+sys.path.append(os.path.join(os.path.dirname(__file__), "../functions/"))
+import pearson
 
 def test_pearson_2d():
     # Test pearson_2d routine

data/.gitignore

@@ -3,3 +3,6 @@
 #data
 data/ds005/
 data/ds005_raw_checksums.txt
+data/ds005_filtered/
+dwn_filtered_data_script.sh
+data/ds114/

data/Makefile

@@ -1,141 +1,26 @@
-download:
+.PHONY: download_data validate_data download_filtered_data download_test_data
+download_data:
 	wget http://openfmri.s3.amazonaws.com/tarballs/ds005_raw.tgz
-	tar xvzf ds005_raw.tgz
+	tar -xvzf ds005_raw.tgz
 
-validate:
+download_filtered_data:
+	variable=Stop python ../data/filtered_data_sh_script.py
+	variable=Stop bash ../data/dwn_filtered_data_script.sh
+
+validate_data:
 	wget http://openfmri.s3.amazonaws.com/tarballs/ds005_raw_checksums.txt
+	python get_ds005_hashes_from_txt.py
 	python data.py
+
+download_test_data:
+	wget -P ../data/ds114/sub009/BOLD/task002_run001/ http://www.jarrodmillman.com/rcsds/_downloads/ds114_sub009_t2r1.nii
+	wget -P ../data/ds114/sub009/behav/task002_run001/ http://www.jarrodmillman.com/rcsds/_downloads/ds114_sub009_t2r1_cond.txt
+	wget -P ../data/ds114/sub009/behav/task002_run001/ http://www.jarrodmillman.com/rcsds/_downloads/ds114_sub009_t2r1_conv.txt
 
-unzip:
-	gzip -d ds005/sub001/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub001/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub001/anatomy/highres001.nii.gz
-	gzip -d ds005/sub001/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub001/anatomy/inplane.nii.gz
-	gzip -d ds005/sub001/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub001/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub001/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub002/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub002/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub002/anatomy/highres001.nii.gz
-	gzip -d ds005/sub002/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub002/anatomy/inplane.nii.gz
-	gzip -d ds005/sub002/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub002/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub002/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub003/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub003/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub003/anatomy/highres001.nii.gz
-	gzip -d ds005/sub003/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub003/anatomy/inplane.nii.gz
-	gzip -d ds005/sub003/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub003/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub003/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub004/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub004/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub004/anatomy/highres001.nii.gz
-	gzip -d ds005/sub004/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub004/anatomy/inplane.nii.gz
-	gzip -d ds005/sub004/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub004/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub004/BOLD/task001_run003/bold.nii.gz		
-	gzip -d ds005/sub005/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub005/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub005/anatomy/highres001.nii.gz
-	gzip -d ds005/sub005/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub005/anatomy/inplane.nii.gz
-	gzip -d ds005/sub005/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub005/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub005/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub006/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub006/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub006/anatomy/highres001.nii.gz
-	gzip -d ds005/sub006/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub006/anatomy/inplane.nii.gz
-	gzip -d ds005/sub006/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub006/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub006/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub007/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub007/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub007/anatomy/highres001.nii.gz
-	gzip -d ds005/sub007/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub007/anatomy/inplane.nii.gz
-	gzip -d ds005/sub007/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub007/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub007/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub008/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub008/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub008/anatomy/highres001.nii.gz
-	gzip -d ds005/sub008/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub008/anatomy/inplane.nii.gz
-	gzip -d ds005/sub008/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub008/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub008/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub009/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub009/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub009/anatomy/highres001.nii.gz
-	gzip -d ds005/sub009/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub009/anatomy/inplane.nii.gz
-	gzip -d ds005/sub009/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub009/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub009/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub010/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub010/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub010/anatomy/highres001.nii.gz
-	gzip -d ds005/sub010/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub010/anatomy/inplane.nii.gz
-	gzip -d ds005/sub010/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub010/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub010/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub011/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub011/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub011/anatomy/highres001.nii.gz
-	gzip -d ds005/sub011/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub011/anatomy/inplane.nii.gz
-	gzip -d ds005/sub011/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub011/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub011/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub012/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub012/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub012/anatomy/highres001.nii.gz
-	gzip -d ds005/sub012/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub012/anatomy/inplane.nii.gz
-	gzip -d ds005/sub012/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub012/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub012/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub013/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub013/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub013/anatomy/highres001.nii.gz
-	gzip -d ds005/sub013/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub013/anatomy/inplane.nii.gz
-	gzip -d ds005/sub013/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub013/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub013/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub014/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub014/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub014/anatomy/highres001.nii.gz
-	gzip -d ds005/sub014/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub014/anatomy/inplane.nii.gz
-	gzip -d ds005/sub014/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub014/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub014/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub015/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub015/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub015/anatomy/highres001.nii.gz
-	gzip -d ds005/sub015/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub015/anatomy/inplane.nii.gz
-	gzip -d ds005/sub015/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub015/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub015/BOLD/task001_run003/bold.nii.gz
-	gzip -d ds005/sub016/anatomy/highres001_brain_mask.nii.gz
-	gzip -d ds005/sub016/anatomy/highres001_brain.nii.gz
-	gzip -d ds005/sub016/anatomy/highres001.nii.gz
-	gzip -d ds005/sub016/anatomy/inplane_brain.nii.gz
-	gzip -d ds005/sub016/anatomy/inplane.nii.gz
-	gzip -d ds005/sub016/BOLD/task001_run001/bold.nii.gz
-	gzip -d ds005/sub016/BOLD/task001_run002/bold.nii.gz	
-	gzip -d ds005/sub016/BOLD/task001_run003/bold.nii.gz
-
+test: 
+	 nosetests tests -w ..
 
+coverage:
+	 nosetests tests -w .. --with-coverage --cover-package=data.py 
 
 

data/data.py

@@ -1,46 +1,22 @@
-from __future__ import print_function, division
+"""data.py
 
-import hashlib
-import os
-import pdb
-
-
-
-def create_dict(filename):
-    newDict={}
-    f=open(filename)
-    num_lines = sum(1 for line in open(filename))
-    for line,i in zip(f,range(0,num_lines)):
-        info = line.split()
-        newDict[info[1]]=info[0]
-    f.close()
-    return newDict
+Simple script to validate the downloaded data:
+ - ds005
+ - ds005_filtered
+ - ds114
+"""
 
-def generate_file_md5(filename, blocksize=2**20):
-    m = hashlib.md5()
-    f= open(filename)
-    while True:
-        buf = f.read(blocksize)
-        #pdb.set_trace()
-        buf.decode('latin-1').encode('utf-8')
-        if not buf:
-            break
-        m.update(buf)
-    f.close()
-    return m.hexdigest()
-
-
-def check_hashes(newDict):
-    all_good = True
-    for k, v in newDict.items():
-        digest = generate_file_md5(k)
-        if v == digest:
-            print("The file {0} has the correct hash.".format(k))
-        else:
-            print("ERROR: The file {0} has the WRONG hash!".format(k))
-            all_good = False
-    return all_good
-
-if __name__=="__main__":
-    newDict = create_dict('ds005_raw_checksums.txt')
-    print(check_hashes(newDict))
+import pdb
+import json
+
+from data_hashes import check_hashes 
+
+if __name__ == "__main__":
+#TODO: add the folder ds005 here
+    file_ls = ['ds005_filtered_hashes.json',\
+              # 'ds005_hashes.json',\
+               'ds114_hashes.json']
+    for f in file_ls:
+        with open(f) as infile:
+            f_dict = json.load(infile)
+            check_hashes(f_dict)