Merge pull request #67 from gitter-lab/github_issues

Fixed regix bug

.github/workflows/install-test.yml

@@ -44,6 +44,8 @@ jobs:
         bash tests/standalone_test.sh
         # Test standalone SINGE on a branching dataset
         bash tests/standalone_branching_test.sh
+        # Test standalone SINGE on a dataset with regulator indices
+        bash tests/standalone_regix_test.sh
         # Confirm the hyperparameter generation script works
         cd scripts
         bash generate_hyperparameters.sh
@@ -93,4 +95,4 @@ jobs:
       run: |
         bash SINGE.sh $MATLAB_RUNTIME_PATH standalone data1/X_SCODE_data.mat data1/gene_list.mat Output tests/example_hyperparameters.txt
         ls -l Output
-        bash tests/compare_example_output.sh Output 1e-02 1e-02
+        bash tests/compare_example_output.sh Output tests/reference/latest 1e-02 1e-02

README.md

@@ -106,7 +106,7 @@ docker run -v $(pwd):/SINGE -w /SINGE agitter/singe:0.4.0 standalone data1/X_SCO
 
 #### Arbitrary commands using Bash as the entry point
 ```
-docker run -v $(pwd):/SINGE -w /SINGE --entrypoint "/bin/bash" agitter/singe:0.4.0 -c "source ~/.bashrc; conda activate singe-test; tests/compare_example_output.sh Output
+docker run -v $(pwd):/SINGE -w /SINGE --entrypoint "/bin/bash" agitter/singe:0.4.0 -c "source ~/.bashrc; conda activate singe-test; tests/compare_example_output.sh Output tests/reference/latest
 ```
 This example is part of the SINGE test code, which only runs when called from the root of the SINGE git repository.
 

code/parseParams.m

@@ -76,8 +76,12 @@
 X = sparse(X);
 % Changed the creation of TempMat to load and save Data file in v7.3 to
 % avoid lower version mat files causing error.
-if exist('branches')
+if exist('branches','var')&&exist('regix','var')
+    save(['TempMat' '_' num2str(params.ID) '.mat'],'X','ptime','branches','regix','-v7.3');
+elseif exist('branches','var')
     save(['TempMat' '_' num2str(params.ID) '.mat'],'X','ptime','branches','-v7.3');
+elseif exist('regix','var')
+    save(['TempMat' '_' num2str(params.ID) '.mat'],'X','ptime','regix','-v7.3');
 else
     save(['TempMat' '_' num2str(params.ID) '.mat'],'X','ptime','-v7.3');
 end

data1/README.md

@@ -1,9 +1,10 @@
 # ESC to Endoderm Differentiation Dataset
 This directory contains the following files related to the ESC to Endoderm Differentiation dataset.
 
-- **gene_list.mat:** list of 100 genes in the dataset.
+- **gene_list.mat:** list of the names of the 100 genes in the dataset.
 - **X_SCODE_data.mat:** Pre-processed version of expression data corresponding to the genes in gene_list.mat from 356 pseudotime-ordered cells.
-- **X_BranchTest.mat:** An alternative version of the expression data that is used only for SINGE test cases.
+- **X_BranchTest.mat:** An alternative version of the expression data that is used only for SINGE test cases with branching processes.
+- **X_regix_test.mat:** An alternative version of the expression data that is used only for SINGE test cases with regulator indices.
 
 The sources to cite for this dataset are:
 - [Matsumoto et al. 2017](https://academic.oup.com/bioinformatics/article/33/15/2314/3100331), [dataset](https://github.com/hmatsu1226/SCODE/tree/master/data)

docker/Dockerfile

@@ -36,20 +36,20 @@ ENV SINGE_RUNNING_IN_DOCKER 1
 
 # Download the compiled SINGE executables from the stable release
 # md5sum of v0.5.0 SINGE_GLG_Test is 1834d44cb686cdfd9b7ee99398518115
-RUN tag=v0.5.0 && \
-    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_Test && \
-    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_GLG_Test && \
-    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_Aggregate && \
-    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/code.md5 && \
-    chmod u+x SINGE_* && \
-    mv SINGE_Test tests/SINGE_Test
+#RUN tag=v0.5.0 && \
+#    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_Test && \
+#    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_GLG_Test && \
+#    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/SINGE_Aggregate && \
+#    wget --quiet https://github.com/gitter-lab/SINGE/releases/download/$tag/code.md5 && \
+#    chmod u+x SINGE_* && \
+#    mv SINGE_Test tests/SINGE_Test
 
 # Download an intermediate version of the compiled SINGE executables for testing
 # Download the md5sums of the source .m files and binaries
-#RUN md5=1834d44cb686cdfd9b7ee99398518115 && \
-#    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_Test && \
-#    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_GLG_Test && \
-#    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_Aggregate && \
-#    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/code.md5 && \
-#    chmod u+x SINGE_* && \
-#    mv SINGE_Test tests/SINGE_Test
+RUN md5=e9bc31dfc6274f4874cc2bde00d85787 && \
+    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_Test && \
+    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_GLG_Test && \
+    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/SINGE_Aggregate && \
+    wget --quiet --no-check-certificate https://www.biostat.wisc.edu/~gitter/tmp/$md5/code.md5 && \
+    chmod u+x SINGE_* && \
+    mv SINGE_Test tests/SINGE_Test

tests/README.md

@@ -6,13 +6,18 @@ Multiple versions of the reference outputs are provided in order to track how th
 
 - `reference/original`: A directory containing the expected outputs from v0.1.0 of SINGE run on the example data after updating them to MAT file format v7.3
 - `reference/latest`: A directory containing the expected outputs from the latest version of SINGE
+- `reference/branching`: A directory containing the expected outputs from the latest version of SINGE for testing branch indices
+- `reference/regulator`: A directory containing the expected outputs from the latest version of SINGE for testing regulator indices
 - `environment.yml`: A conda environment with the Python packages required for the test scripts
 - `example_hyperparameters.txt`: The hyperparameter combinations used to test SINGE
 - `docker_test.sh`: A script that confirms the downloaded compiled code matches the source code, runs SINGE on example data, and compares the output with the reference files inside the Docker container
 - `SINGE_Test.m`: A MATLAB script derived from `SINGE_Example.m` that tests running SINGE through `SINGE.m` instead of the standalone script
 - `high_throughput_test.sh`: A script that simulates running SINGE in a high-throughput computing environment, using Docker to make separate calls to `SINGE.sh` for each GLG run and the aggregation step
 - `standalone_test.sh`: A script uses Docker to run `SINGE.sh` in standalone mode
+- `standalone_branching_test.sh`: A script uses Docker to run `SINGE.sh` in standalone mode when branch indices are provided
+- `standalone_regulator_test.sh`: A script uses Docker to run `SINGE.sh` in standalone mode when regulator indices are provided
 - `compare_example_output.sh`: A script to compare the generated and reference output files
+- `compare_branching_output.sh`: A script to compare the generated and reference output files when branch indices are provided
 - `compare_adj_matrices.py`: A Python script to compare the sparse adjacency matrices in two `.mat` files
 
 #### `compare_adj_matrices.py` usage

tests/compare_example_output.sh

@@ -3,10 +3,11 @@
 # This script must be called from the base directory of the repository
 # Must have already run SINGE on the example data
 # The SINGE output directory is provided as an argument
+# The reference SINGE output directory is provided as an argument
 # Must run inside the conda environment specified by environment.yml or have
 # those Python packages available
 
-usage="Usage: $(basename $0) output-directory [rtol] [atol]"
+usage="Usage: $(basename $0) output-directory reference-directory [rtol] [atol] [dataset]"
 
 if [ $# -gt 0 ]; then
   # First arugment is the SINGE output directory
@@ -20,19 +21,32 @@ else
   exit 1
 fi
 
+if [ $# -gt 1 ]; then
+  # Second arugment is the reference directory with the expected SINGE output
+  refdir=$2
+else
+  echo $usage
+  exit 1
+fi
+
 # Set optional relative and absolute tolerance for comparing adjacency matrix values
 rtol=""
-if [ $# -gt 1 ]; then
-  echo Setting relative tolerance: $2
-  rtol=--rtol=$2
+if [ $# -gt 2 ]; then
+  echo Setting relative tolerance: $3
+  rtol=--rtol=$3
 fi
 atol=""
-if [ $# -gt 2 ]; then
-  echo Setting absolute tolerance: $3
-  atol=--atol=$3
+if [ $# -gt 3 ]; then
+  echo Setting absolute tolerance: $4
+  atol=--atol=$4
 fi
 
-refdir=tests/reference/latest
+# Set optional dataset name
+dataset=X_SCODE_data
+if [ $# -gt 4 ]; then
+  echo Setting dataset name: $5
+  dataset=$5
+fi
 
 # Return 0 unless any individual test fails
 # Continue running all tests even if one fails
@@ -63,7 +77,7 @@ for id in 541 542
 do
   for rep in 1 2
   do
-    filename=AdjMatrix_data1_X_SCODE_datapmat_ID_${id}_lambda_0p01_replicate_${rep}.mat
+    filename=AdjMatrix_data1_${dataset}pmat_ID_${id}_lambda_0p01_replicate_${rep}.mat
     python tests/compare_adj_matrices.py $outdir/$filename $refdir/$filename $rtol $atol
     return_code=$?
     if [[ $return_code -ne 0 ]] ; then

tests/docker_test.sh

@@ -23,7 +23,7 @@ $SINGE_ROOT/SINGE.sh /usr/local/MATLAB/MATLAB_Runtime/v94 standalone data1/X_SCO
 ls script_output/ -l
 
 # Run the tests to compare the SINGE outputs from the standalone script
-tests/compare_example_output.sh script_output
+tests/compare_example_output.sh script_output tests/reference/latest
 
 # Run SINGE on the example data using the compiled SINGE_Test
 echo Testing SINGE with compiled SINGE_Test
@@ -32,4 +32,4 @@ ls compiled_output/ -l
 
 # Run the tests to compare the SINGE outputs from the compiled SINGE_Test
 # The output directory is hard-coded to compiled_output
-tests/compare_example_output.sh compiled_output
+tests/compare_example_output.sh compiled_output tests/reference/latest

tests/high_throughput_test.sh

@@ -24,4 +24,4 @@ ls $output -l
 # Run the tests to evaluate the SINGE outputs from the standalone script
 # run in the high-throughput setting
 docker run -v $(pwd):/SINGE -w /SINGE --entrypoint "/bin/bash" agitter/singe:tmp -c \
-  "source ~/.bashrc; conda activate singe-test; tests/compare_example_output.sh $output"
+  "source ~/.bashrc; conda activate singe-test; tests/compare_example_output.sh $output tests/reference/latest"

tests/reference/regix/SINGE_Gene_Influence.txt

@@ -0,0 +1,101 @@
+Gene_Name	influence
+RARG	1.83513
+SIX1	0.01712
+SOX2	0
+ESRRB	0
+UTF1	0
+EPAS1	0
+FOXQ1	0
+KLF2	0
+GATA4	0
+SOX17	0
+NR0B1	0
+ZFP42	0
+POU5F1	0
+DNMT3A	0
+TFCP2L1	0
+TCF15	0
+ELF3	0
+NANOG	0
+HMGA1	0
+ETV5	0
+CREB3L2	0
+FOXH1	0
+NFXL1	0
+SOX7	0
+TAX1BP3	0
+TET1	0
+JARID2	0
+PEG3	0
+ID2	0
+RBPJ	0
+KDM5B	0
+EGR1	0
+PARP1	0
+BHLHE40	0
+RUNX1	0
+FOXA2	0
+SNAI1	0
+KLF6	0
+BMP2	0
+CREB3	0
+CARHSP1	0
+TGIF1	0
+ZBTB10	0
+XBP1	0
+ZFHX3	0
+DNMT3B	0
+RERE	0
+MSC	0
+TRP53	0
+ZFP57	0
+ZFP710	0
+MYCN	0
+BCL3	0
+GATA6	0
+HOPX	0
+ID1	0
+HNF1B	0
+L3MBTL3	0
+KLF3	0
+DNMT3L	0
+SMAD7	0
+KLF10	0
+TBX3	0
+GLI1	0
+FOXD3	0
+ETV4	0
+MYBL2	0
+TEAD4	0
+NFIL3	0
+HMGA2	0
+GLI2	0
+MSX2	0
+WHSC1	0
+KLF4	0
+MTF2	0
+RHOX6	0
+ID3	0
+ZFP428	0
+REST	0
+PURB	0
+PHB	0
+ZC3H7A	0
+ERF	0
+TET2	0
+SATB2	0
+ZFP296	0
+AFF1	0
+HNRNPK	0
+TCF7L2	0
+ZFP532	0
+ZBTB44	0
+ELF2	0
+ETS1	0
+JUN	0
+BMYC	0
+POU4F2	0
+TFEB	0
+DDIT3	0
+SALL4	0
+ZFP462	0