Merge b506fce into 75e81f0

Clinical-Genomics · Dec 15, 2022 · 29e9242 · 29e9242
2 parents 75e81f0 + b506fce
commit 29e9242
Show file tree

Hide file tree

Showing 18 changed files with 179 additions and 930 deletions.
diff --git a/.coverage b/.coverage
diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml
@@ -1,39 +1,35 @@
 ---
-
 name: Demultiplexing CI
 
 "on": ["push", "pull_request"]
 
 jobs:
-
-  tests-coverage:
-
-    runs-on: ubuntu-latest
-
-    steps:
-
-    # Check out Demultiplexing code
-    - uses: actions/checkout@v2
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.7
-
-    - name: Set up poetry
-      uses: abatilo/actions-poetry@v2.0.0
-      with:
-        poetry-version: 1.1.4
-
-    - name: Demultiplexing dependencies
-      run: |
-        poetry install
-
-    - name: Test with pytest & Coveralls
-      run: |
-        poetry run pytest --cov=demux/
-        poetry run coveralls
-        bash tests/test.bash
-      env:
-        GITHUB: 1
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+    tests-coverage:
+        runs-on: ubuntu-latest
+
+        steps:
+            # Check out Demultiplexing code
+            - uses: actions/checkout@v2
+
+            - name: Set up Python 3.7
+              uses: actions/setup-python@v2
+              with:
+                  python-version: 3.7
+
+            - name: Set up poetry
+              uses: abatilo/actions-poetry@v2.0.0
+              with:
+                  poetry-version: 1.3.1
+
+            - name: Demultiplexing dependencies
+              run: |
+                  poetry install
+
+            - name: Test with pytest & Coveralls
+              run: |
+                  poetry run pytest --cov=demux/
+                  poetry run coveralls
+                  bash tests/test.bash
+              env:
+                  GITHUB: 1
+                  COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
diff --git a/.github/workflows/installations.yml b/.github/workflows/installations.yml
@@ -1,47 +1,38 @@
 ---
-
 name: Demultiplex installation tests
 
 "on": ["pull_request"]
 
 jobs:
-
-  installations:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7]
-
-    steps:
-
-    - name: Check out git repository
-      uses: actions/checkout@v2
-
-    # Set up python
-    - name: Set up Python ${{ matrix.python-version}}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version}}
-
-    - name: Install pip
-      # Make sure that we have correct python version and pip version
-      # For details see .github/workflows/server_info.md
-      run: |
-        if [ "$PYTHON_VERSION" = "$OLD_VERSION" ]; then
-          python -m pip install --upgrade pip==9.0.1
-        else
-          python -m pip install --upgrade pip==18.1
-        fi
-        echo $PYTHON_VERSION
-        pip --version
-      env:
-        PYTHON_VERSION: ${{ matrix.python-version}}
-        OLD_VERSION: 3.6
-
-    - name: Production Dependencies
-      run: |
-        pip install micropipenv[toml]
-        micropipenv requirements --method poetry > requirements.txt
-        pip install -r requirements.txt
-        pip check
+    installations:
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                python-version: [3.7]
+
+        steps:
+            - name: Check out git repository
+              uses: actions/checkout@v2
+
+            # Set up python
+            - name: Set up Python ${{matrix.python-version}}
+              uses: actions/setup-python@v2
+              with:
+                  python-version: ${{matrix.python-version}}
+
+            - name: Install pip
+              # Make sure that we have correct python version and pip version
+              # For details see .github/workflows/server_info.md
+              run: |
+                  python -m pip install --upgrade pip==18.1
+                  echo $PYTHON_VERSION
+                  pip --version
+              env:
+                  PYTHON_VERSION: ${{matrix.python-version}}
+
+            - name: Production Dependencies
+              run: |
+                  pip install micropipenv[toml]
+                  micropipenv requirements --method poetry > requirements.txt
+                  pip install -r requirements.txt
+                  pip check
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ DEVELOP/
 .idea/
 .vscode
 .venv
+venv
 
 # Distribution / packaging
 *.egg-info/

diff --git a/scripts/2500/checkfornewrun.bash b/scripts/2500/checkfornewrun.bash
@@ -5,6 +5,17 @@ shopt -s nullglob
 INDIR=${1?'please provide a run dir'}
 DEMUXDIR=${2?'please provide a demux dir'}
 
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV_BASE="${CONDA_BASE}/envs"
+CONDA_ENV="S_demux"
+
+if [[ ${ENVIRONMENT} == 'production' ]]; then
+    CONDA_ENV="P_demux"
+fi
+
+
+
 for RUNDIR in ${INDIR}/*; do
     RUN=$(basename ${RUNDIR})
     NOW=$(date +"%Y%m%d%H%M%S")
@@ -18,19 +29,19 @@ for RUNDIR in ${INDIR}/*; do
                 if grep -qs ',ctmr,' ${RUNDIR}/SampleSheet.csv; then
                     echo [${NOW}] ${RUN} is CTMR - transmogrifying SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/SampleSheet.ctmr
-                    demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
+                    $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
                 fi
             fi
             if [[ ! -e ${RUNDIR}/SampleSheet.csv ]]; then
                 echo [${NOW}] ${RUN} fetching samplesheet.csv
                 FC=${RUN##*_}
                 FC=${FC:1}
-                demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
+                $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
                 cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
             else
                 echo converting ${RUNDIR}/SampleSheet.csv
-                demux sheet convert ${RUNDIR}/SampleSheet.csv > ${RUNDIR}/SampleSheet.conv
+                $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet convert ${RUNDIR}/SampleSheet.csv > ${RUNDIR}/SampleSheet.conv
                 cp ${RUNDIR}/SampleSheet.conv ${RUNDIR}/SampleSheet.csv
                 cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
             fi

diff --git a/scripts/2500/demux-2500.bash b/scripts/2500/demux-2500.bash
@@ -22,6 +22,17 @@ RUN=$(basename ${IN_DIR})
 RUN_DIR=$(dirname ${IN_DIR})
 PROJECTLOG=${OUT_DIR}/${RUN}/projectlog.$(date +"%Y%m%d%H%M%S").log
 
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV_BASE="${CONDA_BASE}/envs"
+CONDA_ENV="S_demux"
+
+if [[ ${ENVIRONMENT} == 'production' ]]; then
+    CONDA_ENV="P_demux"
+fi
+
+
+
 SCRIPT_DIR=/home/proj/${ENVIRONMENT}/bin/git/demultiplexing/scripts/2500/
 
 SLURM_ACCOUNT=development
@@ -73,7 +84,7 @@ cp ${IN_DIR}/SampleSheet.csv ${IN_DIR}/Data/Intensities/BaseCalls/SampleSheet.cs
 log "Setup correct, starts demuxing . . ."
 
 echo "get basemask ${IN_DIR}"
-BASEMASK=$(demux basemask create --application wes --lane ${LANE} ${IN_DIR})
+BASEMASK=$($CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux basemask create --application wes --lane ${LANE} ${IN_DIR})
 UNALIGNED_DIR=Unaligned-${BASEMASK//,}
 
 # DEMUX !

diff --git a/scripts/2500/stage/checkfornewrun.bash b/scripts/2500/stage/checkfornewrun.bash
@@ -5,6 +5,21 @@ shopt -s nullglob
 INDIR=${1?'please provide a run dir'}
 DEMUXDIR=${2?'please provide a demux dir'}
 
+
+INDIR=${1?'please provide a run dir'}
+DEMUXDIR=${2?'please provide a demux dir'}
+
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV_BASE="${CONDA_BASE}/envs"
+CONDA_ENV="S_demux"
+
+if [[ ${ENVIRONMENT} == 'production' ]]; then
+    CONDA_ENV="P_demux"
+fi
+
+
+
 for RUNDIR in ${INDIR}/*; do
     RUN=$(basename ${RUNDIR})
     NOW=$(date +"%Y%m%d%H%M%S")
@@ -18,19 +33,19 @@ for RUNDIR in ${INDIR}/*; do
                 if grep -qs ',ctmr,' ${RUNDIR}/SampleSheet.csv; then
                     echo [${NOW}] ${RUN} is CTMR - transmogrifying SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/SampleSheet.ctmr
-                    demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
+                    $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
                 fi
             fi
             if [[ ! -e ${RUNDIR}/SampleSheet.csv ]]; then
                 echo [${NOW}] ${RUN} fetching samplesheet.csv
                 FC=${RUN##*_}
                 FC=${FC:1}
-                demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
+                $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
                 cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
             else
                 echo converting ${RUNDIR}/SampleSheet.csv
-                demux sheet convert ${RUNDIR}/SampleSheet.csv > ${RUNDIR}/SampleSheet.conv
+                $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet convert ${RUNDIR}/SampleSheet.csv > ${RUNDIR}/SampleSheet.conv
                 cp ${RUNDIR}/SampleSheet.conv ${RUNDIR}/SampleSheet.csv
                 cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
             fi

diff --git a/scripts/2500/stage/demux-2500.bash b/scripts/2500/stage/demux-2500.bash
@@ -24,9 +24,17 @@ PROJECTLOG=${OUT_DIR}/${RUN}/projectlog.$(date +"%Y%m%d%H%M%S").log
 
 SCRIPT_DIR=/home/proj/${ENVIRONMENT}/bin/git/demultiplexing/scripts/2500/stage/        # use this when testing on stage
 
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV_BASE="${CONDA_BASE}/envs"
+CONDA_ENV="S_demux"
+
+
+
 SLURM_ACCOUNT=development
 if [[ ${ENVIRONMENT} == 'production' ]]; then
     SLURM_ACCOUNT=production
+    CONDA_ENV="P_demux"
 fi
 
 #############
@@ -73,7 +81,7 @@ cp ${IN_DIR}/SampleSheet.csv ${IN_DIR}/Data/Intensities/BaseCalls/SampleSheet.cs
 log "Setup correct, starts demuxing . . ."
 
 echo "get basemask ${IN_DIR}"
-BASEMASK=$(demux basemask create --application wes --lane ${LANE} ${IN_DIR})
+BASEMASK=$($CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux basemask create --application wes --lane ${LANE} ${IN_DIR})
 UNALIGNED_DIR=Unaligned-${BASEMASK//,}
 
 # DEMUX !

diff --git a/scripts/checkfornewrun.bash b/scripts/checkfornewrun.bash
@@ -7,6 +7,20 @@ source ~/.aliases
 INDIR=${1?'please provide a run dur'}
 DEMUXDIR=${2?'please provide a demux dir'}
 
+### SETUP CONDA VARIABLES ###
+
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV="S_demux"
+
+if [[ ${ENVIRONMENT} == 'production' ]]; then
+    SLURM_ACCOUNT=production
+    CONDA_ENV="P_demux"
+fi
+
+CONDA_ENV_BIN_BASE="${CONDA_BASE}/envs/${CONDA_ENV}/bin/"
+
+
 for RUNDIR in ${INDIR}/*; do
     RUN=$(basename ${RUNDIR})
     NOW=$(date +"%Y%m%d%H%M%S")
@@ -20,15 +34,15 @@ for RUNDIR in ${INDIR}/*; do
                 if grep -qs ',ctmr,' ${RUNDIR}/SampleSheet.csv; then
                     echo [${NOW}] ${RUN} is CTMR - transmogrifying SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/SampleSheet.ctmr
-                    demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
+                    $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet demux -a miseq ${RUNDIR}/SampleSheet.ctmr > ${RUNDIR}/SampleSheet.csv
                     cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
                 fi
             fi
             if [[ ! -e ${RUNDIR}/SampleSheet.csv ]]; then
                 echo [${NOW}] ${RUN} fetching samplesheet.csv
                 FC=${RUN##*_}
                 FC=${FC:1}
-                demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
+                $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet fetch --application wes --shortest ${FC} > ${RUNDIR}/SampleSheet.csv
                 cp ${RUNDIR}/SampleSheet.csv ${RUNDIR}/Data/Intensities/BaseCalls/
             fi
             echo [${NOW}] ${RUN} starting demultiplexing

diff --git a/scripts/hiseqx/xdemuxtiles.bash b/scripts/hiseqx/xdemuxtiles.bash
@@ -7,7 +7,7 @@ set -eu -o pipefail
 # PARAMS #
 ##########
 
-VERSION=5.4.3
+VERSION=5.5.0
 RUNDIR=${1?'full path to run dir'}
 OUTDIR=${2-"/home/proj/${ENVIRONMENT}/demultiplexed-runs/$(basename "${RUNDIR}")/"}
 
@@ -16,10 +16,21 @@ LOGDIR="${OUTDIR}/LOG"
 SCRIPTDIR="$(dirname "$(readlink -nm "$0")")"
 
 SLURM_ACCOUNT=development
+
+### SETUP CONDA VARIABLES ###
+
+CONDA_BASE="/home/proj/${ENVIRONMENT}/bin/miniconda3"
+CONDA_EXE="${CONDA_BASE}/bin/conda"
+CONDA_ENV="S_demux"
+
 if [[ ${ENVIRONMENT} == 'production' ]]; then
     SLURM_ACCOUNT=production
+    CONDA_ENV="P_demux"
 fi
 
+CONDA_ENV_BIN_BASE="${CONDA_BASE}/envs/${CONDA_ENV}/bin/"
+
+
 #############
 # FUNCTIONS #
 #############
@@ -55,7 +66,7 @@ mkdir -p "${LOGDIR}"
 log "demuxtiles.bash VERSION ${VERSION}"
 
 # get the flowcell name
-FC=$( basename "$(basename "${RUNDIR}"/)" | awk 'BEGIN {FS="/"} {split($(NF-1),arr,"_");print substr(arr[4],2,length(arr[4]))}')
+FC=$(basename "$(basename "${RUNDIR}"/)" | awk 'BEGIN {FS="/"} {split($(NF-1),arr,"_");print substr(arr[4],2,length(arr[4]))}')
 
 # is this a dual-index FC?
 IS_DUAL=$(grep IndexRead2 "${RUNDIR}/runParameters.xml" | sed 's/<\/IndexRead2>\r//' | sed 's/    <IndexRead2>//')
@@ -73,12 +84,12 @@ if [[ ! -e ${RUNDIR}/SampleSheet.csv ]]; then
     if [[ ${IS_DUAL} == '8' ]]; then
         DUALINDEX_PARAM='--dualindex'
     fi
-    log "demux sheet fetch -a wgs ${DUALINDEX_PARAM} ${FC} > ${RUNDIR}/SampleSheet.csv"
-    demux sheet fetch -a wgs ${DUALINDEX_PARAM} "${FC}" > "${RUNDIR}/SampleSheet.csv"
+    log "${CONDA_RUN_COMMAND}/demux sheet fetch -a wgs ${DUALINDEX_PARAM} ${FC} > ${RUNDIR}/SampleSheet.csv"
+    $CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet fetch -a wgs $DUALINDEX_PARAM "${FC}" > "${RUNDIR}/SampleSheet.csv"
 fi
 
 # validate!
-demux sheet validate --application wgs "${RUNDIR}/SampleSheet.csv"
+$CONDA_EXE run --name $CONDA_ENV $CONDA_ENV_BIN_BASE/demux sheet validate --application wgs "${RUNDIR}/SampleSheet.csv"
 
 # notify we are ready to start!
 mail -s "DEMUX of $FC started" ${EMAIL} < "${RUNDIR}/SampleSheet.csv"