Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(DO NOT MERGE) Added non-root user to Dockerfile. #6525

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

(DO NOT MERGE) Added non-root user to Dockerfile. #6525

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7959d6
Changed default location of gatk.jar in WDLs from /root to /gatk.
samuelklee Mar 25, 2020
26766c3
Added non-root user to Dockerfile.
samuelklee Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,28 +137,28 @@ script:
echo "Deleting some unused files before running M2 WDL...";
rm -Rf src/test/resources/large/VQSR;
echo "Running M2 WDL";
sudo bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
elif [[ $RUN_CNN_WDL == true ]]; then
echo "Running CNN WDL";
sudo bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
elif [[ $TEST_DOCKER == true ]]; then
echo "Building docker image and running appropriate tests..." ;
if [ ${TRAVIS_PULL_REQUEST} != false ]; then
sudo bash build_docker.sh -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
bash build_docker.sh -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
DOCKER_TAG=FETCH_HEAD;
else
echo ${TRAVIS_COMMIT};
sudo bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$TRAVIS_COMMIT;
fi;
sudo docker images;
docker images;
echo ${TEST_TYPE};
sudo mkdir -p build/reports/;
sudo chmod -R a+w build/reports/;
mkdir -p build/reports/;
chmod -R a+w build/reports/;
cp scripts/docker/dockertest.gradle .;
sudo docker run -v $(pwd):/gatkCloneMountPoint:cached -v $(pwd)/testJars:/jars:cached --rm -e "scala.version=${SCALA_VERSION}" -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
docker run -v $(pwd):/gatk/gatkCloneMountPoint -v $(pwd)/testJars:/gatk/jars --rm -e "scala.version=${SCALA_VERSION}" -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /gatk/run_unit_tests.sh;
TEST_EXIT_VALUE=$?;
sudo mkdir build/reports/tests/test && sudo cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && sudo rm -r build/reports/tests/testOnPackagedReleaseJar;
mkdir build/reports/tests/test && cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && rm -r build/reports/tests/testOnPackagedReleaseJar;
$( exit ${TEST_EXIT_VALUE} );
else
./gatk PrintReads -I src/test/resources/NA12878.chr17_69k_70k.dictFix.bam -O output.bam;
Expand Down
65 changes: 33 additions & 32 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,68 +1,69 @@
# Using OpenJDK 8
FROM broadinstitute/gatk:gatkbase-2.2.0

ARG UNAME=gatk
ARG UID=1000
ARG GID=1000
# Location of the unzipped gatk bundle files
ARG ZIPPATH

#RUN cp -r install_R_packages.R /gatk

ADD $ZIPPATH /gatk
RUN mkdir /gatk/gatkCloneMountPoint && \
mkdir /gatk/jars && \
mkdir /gatk/.gradle

RUN groupadd -g $GID -o $UNAME
RUN useradd -b / -u $UID -g $GID -o -s /bin/bash $UNAME
RUN chown -R $UID:$GID /gatk
USER $UNAME
ENV HOME /gatk

WORKDIR /gatk

#Setup linked jars that may be needed for running gatk
RUN ln -s $( find /gatk -name "gatk*local.jar" ) gatk.jar
RUN ln -s $( find /gatk -name "gatk*local.jar" ) /root/gatk.jar
RUN ln -s $( find /gatk -name "gatk*spark.jar" ) gatk-spark.jar

WORKDIR /root

# Make sure we can see a help message
# Make sure we can see a help message
RUN java -jar gatk.jar -h
RUN mkdir /gatkCloneMountPoint
RUN mkdir /jars
RUN mkdir .gradle

WORKDIR /gatk

# Create a simple unit test runner
ENV CI true
RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
echo "export GATK_DOCKER_CONTAINER=true" >> /root/run_unit_tests.sh && \
echo "export TEST_JAR=\$( find /jars -name \"gatk*test.jar\" )" >> /root/run_unit_tests.sh && \
echo "export TEST_DEPENDENCY_JAR=\$( find /jars -name \"gatk*testDependencies.jar\" )" >> /root/run_unit_tests.sh && \
echo "export GATK_JAR=$( find /gatk -name "gatk*local.jar" )" >> /root/run_unit_tests.sh && \
echo "mkdir /gatk/srcdir" >> /root/run_unit_tests.sh && \
echo "cp -rp /gatkCloneMountPoint/src/main/java/* /gatk/srcdir" >> /root/run_unit_tests.sh && \
echo "export SOURCE_DIR=/gatk/srcdir" >> /root/run_unit_tests.sh && \
echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false\"" /root/run_unit_tests.sh && \
echo "export CP_DIR=/gatk/testClasses" /root/run_unit_tests.sh && \
echo "ln -s /gatkCloneMountPoint/src/ /gatkCloneMountPoint/scripts/docker/src" >> /root/run_unit_tests.sh && \
echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \
echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
RUN echo "source activate gatk" > /gatk/run_unit_tests.sh && \
echo "export GATK_DOCKER_CONTAINER=true" >> /gatk/run_unit_tests.sh && \
echo "export TEST_JAR=\$( find /gatk/jars -name \"gatk*test.jar\" )" >> /gatk/run_unit_tests.sh && \
echo "export TEST_DEPENDENCY_JAR=\$( find /gatk/jars -name \"gatk*testDependencies.jar\" )" >> /gatk/run_unit_tests.sh && \
echo "export GATK_JAR=$( find /gatk -name "gatk*local.jar" )" >> /gatk/run_unit_tests.sh && \
echo "mkdir /gatk/srcdir" >> /gatk/run_unit_tests.sh && \
echo "cp -rp /gatk/gatkCloneMountPoint/src/main/java/* /gatk/srcdir" >> /gatk/run_unit_tests.sh && \
echo "export SOURCE_DIR=/gatk/srcdir" >> /gatk/run_unit_tests.sh && \
echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false\"" /gatk/run_unit_tests.sh && \
echo "export CP_DIR=/gatk/testClasses" /gatk/run_unit_tests.sh && \
echo "ln -s /gatk/gatkCloneMountPoint/src/ /gatk/gatkCloneMountPoint/scripts/docker/src" >> /gatk/run_unit_tests.sh && \
echo "ln -s /gatk/gatkCloneMountPoint/build/ /gatk/gatkCloneMountPoint/scripts/docker/build" >> /gatk/run_unit_tests.sh && \
echo "cd /gatk/ && /gatk/gatkCloneMountPoint/gradlew -b /gatk/gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatk/gatkCloneMountPoint" >> /gatk/run_unit_tests.sh

WORKDIR /root
RUN cp -r /root/run_unit_tests.sh /gatk
RUN cp -r gatk.jar /gatk
ENV CLASSPATH /gatk/gatk.jar:$CLASSPATH
RUN cp -r install_R_packages.R /gatk

# Start GATK Python environment

ENV DOWNLOAD_DIR /downloads
ENV DOWNLOAD_DIR /gatk/downloads
ENV CONDA_URL https://repo.continuum.io/miniconda/Miniconda3-4.3.30-Linux-x86_64.sh
ENV CONDA_MD5 = "0b80a152332a4ce5250f3c09589c7a81"
ENV CONDA_PATH /opt/miniconda
ENV CONDA_MD5 "0b80a152332a4ce5250f3c09589c7a81"
ENV CONDA_PATH /gatk/miniconda
RUN mkdir $DOWNLOAD_DIR && \
wget -nv -O $DOWNLOAD_DIR/miniconda.sh $CONDA_URL && \
test "`md5sum $DOWNLOAD_DIR/miniconda.sh | awk -v FS=' ' '{print $1}'` = $CONDA_MD5" && \
bash $DOWNLOAD_DIR/miniconda.sh -p $CONDA_PATH -b && \
rm $DOWNLOAD_DIR/miniconda.sh
WORKDIR /gatk
rm -rf $DOWNLOAD_DIR
ENV PATH $CONDA_PATH/envs/gatk/bin:$CONDA_PATH/bin:$PATH
RUN conda env create -n gatk -f /gatk/gatkcondaenv.yml && \
echo "source activate gatk" >> /gatk/gatkenv.rc && \
echo "source /gatk/gatk-completion.sh" >> /gatk/gatkenv.rc && \
conda clean -y -all && \
rm -rf /root/.cache/pip
rm -rf /gatk/.cache/pip

CMD ["bash", "--init-file", "/gatk/gatkenv.rc"]

Expand Down
8 changes: 4 additions & 4 deletions build_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,9 @@ mv $( find ./build/libs/ -name "gatk*testDependencies.jar" ) ${STAGING_ABSOLUTE_

echo "Building image to tag ${REPO_PRJ}:${GITHUB_TAG}..."
if [ -n "${IS_PUSH}" ]; then
docker build -t ${REPO_PRJ}:${GITHUB_TAG} --squash --build-arg ZIPPATH=./unzippedJar .
docker build -t ${REPO_PRJ}:${GITHUB_TAG} --squash --build-arg UID=$(whoami) --build-arg UID=$(id -u) --build-arg GID=$(id -g) --build-arg ZIPPATH=./unzippedJar .
else
docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg ZIPPATH=./unzippedJar .
docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg UID=$(whoami) --build-arg UID=$(id -u) --build-arg GID=$(id -g) --build-arg ZIPPATH=./unzippedJar .
fi

if [ -z "${IS_NOT_RUN_UNIT_TESTS}" ] ; then
Expand All @@ -144,8 +144,8 @@ if [ -z "${IS_NOT_RUN_UNIT_TESTS}" ] ; then
cp build.gradle build.gradle.backup
cp scripts/docker/dockertest.gradle .

echo docker run ${REMOVE_CONTAINER_STRING} -v ${STAGING_ABSOLUTE_PATH}:/gatkCloneMountPoint -v ${STAGING_ABSOLUTE_PATH}/testJars:/jars -t ${REPO_PRJ}:${GITHUB_TAG} bash /root/run_unit_tests.sh
docker run ${REMOVE_CONTAINER_STRING} -v ${STAGING_ABSOLUTE_PATH}:/gatkCloneMountPoint -v ${STAGING_ABSOLUTE_PATH}/testJars:/jars -t ${REPO_PRJ}:${GITHUB_TAG} bash /root/run_unit_tests.sh
echo docker run ${REMOVE_CONTAINER_STRING} -v ${STAGING_ABSOLUTE_PATH}:/gatk/gatkCloneMountPoint -v ${STAGING_ABSOLUTE_PATH}/testJars:/gatk/jars -t ${REPO_PRJ}:${GITHUB_TAG} bash /gatk/run_unit_tests.sh
docker run ${REMOVE_CONTAINER_STRING} -v ${STAGING_ABSOLUTE_PATH}:/gatk/gatkCloneMountPoint -v ${STAGING_ABSOLUTE_PATH}/testJars:/gatk/jars -t ${REPO_PRJ}:${GITHUB_TAG} bash /gatk/run_unit_tests.sh
echo " Unit tests passed..."
mv build.gradle.backup build.gradle
fi
Expand Down
10 changes: 5 additions & 5 deletions scripts/cnn_variant_wdl/cnn_variant_common_tasks.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ task CNNScoreVariants {
command <<<

set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
CNNScoreVariants \
Expand Down Expand Up @@ -98,7 +98,7 @@ task RunHC4 {

command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
HaplotypeCaller \
Expand Down Expand Up @@ -160,7 +160,7 @@ task FilterVariantTranches {

command <<<
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
FilterVariantTranches \
Expand Down Expand Up @@ -213,7 +213,7 @@ task SplitIntervals {

command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
SplitIntervals \
Expand Down Expand Up @@ -261,7 +261,7 @@ task MergeVCFs {

command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}
gatk --java-options "-Xmx${command_mem}m" MergeVcfs \
-I ${sep=' -I ' input_vcfs} -O "${output_vcf}"
}
Expand Down
10 changes: 5 additions & 5 deletions scripts/cnn_variant_wdl/cram2model.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,9 @@ task WriteTensors {
Int command_mem = machine_mem - 1000
command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

mkdir "/root/tensors/"
mkdir "/gatk/tensors/"

gatk --java-options "-Xmx${command_mem}m" \
CNNVariantWriteTensors \
Expand All @@ -170,10 +170,10 @@ task WriteTensors {
-truth-vcf ${truth_vcf} \
-truth-bed ${truth_bed} \
-tensor-type ${tensor_type} \
-output-tensor-dir "/root/tensors/" \
-output-tensor-dir "/gatk/tensors/" \
-bam-file ${input_bam}

tar -czf "tensors.tar.gz" "/root/tensors/"
tar -czf "tensors.tar.gz" "/gatk/tensors/"
}

output {
Expand Down Expand Up @@ -208,7 +208,7 @@ task TrainModel {
Int command_mem = machine_mem - 1000
command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

for tensors in ${sep=' ' tar_tensors} ; do
tar -xzf $tensors
Expand Down
4 changes: 2 additions & 2 deletions scripts/cnn_variant_wdl/variant_classifier_plots.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ task MakeTables {

command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
UpdateVcfSequenceDictionary \
Expand Down Expand Up @@ -205,7 +205,7 @@ task MakeTableNoTruth {

command {
set -e
export GATK_LOCAL_JAR=${default="/root/gatk.jar" gatk_override}
export GATK_LOCAL_JAR=${default="/gatk/gatk.jar" gatk_override}

gatk --java-options "-Xmx${command_mem}m" \
VariantsToTable \
Expand Down
14 changes: 7 additions & 7 deletions scripts/cnv_wdl/cnv_common_tasks.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ task PreprocessIntervals {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

gatk --java-options "-Xmx~{command_mem_mb}m" PreprocessIntervals \
~{"-L " + intervals} \
Expand Down Expand Up @@ -84,7 +84,7 @@ task AnnotateIntervals {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

gatk --java-options "-Xmx~{command_mem_mb}m" AnnotateIntervals \
-L ~{intervals} \
Expand Down Expand Up @@ -145,7 +145,7 @@ task FilterIntervals {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

gatk --java-options "-Xmx~{command_mem_mb}m" FilterIntervals \
-L ~{intervals} \
Expand Down Expand Up @@ -230,7 +230,7 @@ task CollectCounts {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

case ~{format_} in
HDF5 | TSV | TSV_GZ)
Expand Down Expand Up @@ -313,7 +313,7 @@ task CollectAllelicCounts {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

gatk --java-options "-Xmx~{command_mem_mb}m" CollectAllelicCounts \
-L ~{common_sites} \
Expand Down Expand Up @@ -370,7 +370,7 @@ task ScatterIntervals {
set -eu
# IntervalListTools will fail if the output directory does not exist, so we create it
mkdir ~{output_dir_}
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

# IntervalListTools behaves differently when scattering to a single or multiple shards, so we do some handling in bash

Expand Down Expand Up @@ -448,7 +448,7 @@ task PostprocessGermlineCNVCalls {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

sharded_interval_lists_array=(~{sep=" " sharded_interval_lists})

Expand Down
4 changes: 2 additions & 2 deletions scripts/cnv_wdl/germline/cnv_germline_case_workflow.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ task DetermineGermlineContigPloidyCaseMode {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}
export MKL_NUM_THREADS=~{default=8 cpu}
export OMP_NUM_THREADS=~{default=8 cpu}

Expand Down Expand Up @@ -393,7 +393,7 @@ task GermlineCNVCallerCaseMode {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}
export MKL_NUM_THREADS=~{default=8 cpu}
export OMP_NUM_THREADS=~{default=8 cpu}

Expand Down
4 changes: 2 additions & 2 deletions scripts/cnv_wdl/germline/cnv_germline_cohort_workflow.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ task DetermineGermlineContigPloidyCohortMode {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}
export MKL_NUM_THREADS=~{default=8 cpu}
export OMP_NUM_THREADS=~{default=8 cpu}

Expand Down Expand Up @@ -517,7 +517,7 @@ task GermlineCNVCallerCohortMode {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}
export MKL_NUM_THREADS=~{default=8 cpu}
export OMP_NUM_THREADS=~{default=8 cpu}

Expand Down
2 changes: 1 addition & 1 deletion scripts/cnv_wdl/somatic/cnv_somatic_funcotate_seg_workflow.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ task FuncotateSegments {

command <<<
set -eu
export GATK_LOCAL_JAR=~{default="/root/gatk.jar" gatk4_jar_override}
export GATK_LOCAL_JAR=~{default="/gatk/gatk.jar" gatk4_jar_override}

# Extract our data sources:
echo "Extracting data sources zip file..."
Expand Down
Loading