From 394f2aed7513942910d7780acacab3984d8936c2 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 29 Apr 2016 10:30:28 -0700 Subject: [PATCH 1/9] install/cleanup tool_src in single step to reduce image size --- docker/base/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index d51bd8a..f95e004 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -39,8 +39,7 @@ ENV PATH $PATH:$JAVA_HOME/bin # this is not the ideal way to install all these tools, but the code is currently shared by non-docker usages RUN svn co --username cpas --password cpas --no-auth-cache https://hedgehog.fhcrc.org/tor/stedi/branches/${SVN_BRANCH}/externalModules/labModules/SequenceAnalysis/pipeline_code ${LK_HOME}/svn/ -RUN ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -RUN rm -Rf ${LK_HOME}/tool_src +RUN ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -c # Clean up RUN rm -rf /var/lib/apt/lists/* From 9d3d8b4809ab15778af6adac93f499c8bcf5c2b9 Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 3 May 2016 09:04:03 -0700 Subject: [PATCH 2/9] attempt to flatten dockerfiles to save space on intermediate containers --- docker/base/Dockerfile | 42 ++++++++++++++---------------------------- docker/remote/Dockerfile | 41 +++++++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 44 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index f95e004..fa60794 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -16,36 +16,22 @@ ENV LK_HOME /labkey # # Create directories required for running LabKey Server # -run mkdir -p /labkey/ -run mkdir -p /labkey/bin -run mkdir -p /labkey/svn +RUN mkdir -p /labkey/ +RUN mkdir -p /labkey/bin +RUN mkdir -p /labkey/svn -# Update the Ubuntu and install required tools -RUN (apt-get update; \ - apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common; \ - apt-get clean -y) - -# Install Java -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ +# Do all updates and install java in single step to slim down image +# Sequence tools are installed using a separate shell script since this config is shared by non-docker usages +RUN (echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ - apt-get update && \ - apt-get install -y oracle-java8-installer && \ - rm -rf /var/cache/oracle-jdk8-installer - -#rm -rf /var/lib/apt/lists/* && \ + apt-get update; \ + apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common oracle-java8-installer; \ + rm -rf /var/cache/oracle-jdk8-installer && \ + svn co --username cpas --password cpas --no-auth-cache https://hedgehog.fhcrc.org/tor/stedi/branches/${SVN_BRANCH}/externalModules/labModules/SequenceAnalysis/pipeline_code ${LK_HOME}/svn/ && \ + ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -c && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + apt-get clean -y) ENV JAVA_HOME /usr/lib/jvm/java-8-oracle -ENV PATH $PATH:$JAVA_HOME/bin - -# this is not the ideal way to install all these tools, but the code is currently shared by non-docker usages -RUN svn co --username cpas --password cpas --no-auth-cache https://hedgehog.fhcrc.org/tor/stedi/branches/${SVN_BRANCH}/externalModules/labModules/SequenceAnalysis/pipeline_code ${LK_HOME}/svn/ -RUN ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -c - -# Clean up -RUN rm -rf /var/lib/apt/lists/* - -#note: need to sort out users/permissions -#-u labkey -#chown -R labkey:labkey $LK_HOME - +ENV PATH $LK_HOME/bin:$JAVA_HOME/bin:$PATH diff --git a/docker/remote/Dockerfile b/docker/remote/Dockerfile index 5d3b6ba..34a2250 100644 --- a/docker/remote/Dockerfile +++ b/docker/remote/Dockerfile @@ -15,23 +15,32 @@ ENV SVN_BRANCH discvr16.1 ENV LK_HOME /labkey # -# Create directories required for running LabKey Server -# +# This creates a container intended to run as a remote pipeline server. You must provide the config files, +# which should at minimum include a pipelineConfig.xml file. You will likely also need to mount the +# shared folder where you data resides and consider setting an activemq host entry. However, these are all +# configured via pipelineConfig.xml so you do have flexibility. +# +# Example Command: +# +# docker run --rm -v /:/labkey/configs -v /:/labkey_data bbimber/discvr-seq: +# +# Consider adding --add-host activemq:XXX.XXX.XXX.XXX, depending on how pipelineConfig.xml is set up +# + run mkdir -p /labkey/ run mkdir -p /labkey/configs # download latest build -RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/${GZ_PREFIX}-{build.number}-${DIST_NAME}-bin.tar.gz -RUN mv ./teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/*.gz ./ -RUN rm -Rf ./teamcity.labkey.org - -RUN (GZ=$(ls -tr | grep '^LabKey.*\.gz$' | tail -n -1); \ - tar -xf $GZ; \ - DIR=$(echo $GZ | sed -e "s/.tar.gz$//"); \ - cp -R ${DIR}/bin $LK_HOME; \ - cp -R ${DIR}/modules $LK_HOME; \ - cp -R ${DIR}/labkeywebapp $LK_HOME; \ - cp -R ${DIR}/pipeline-lib $LK_HOME; \ - mkdir -p /labkey/apps/tomcat/lib/; \ - cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/; \ - rm -rf ${DIR}) +RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/${GZ_PREFIX}-{build.number}-${DIST_NAME}-bin.tar.gz && \ + mv ./teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/*.gz ./ && \ + rm -Rf ./teamcity.labkey.org && \ + GZ=$(ls -tr | grep '^LabKey.*\.gz$' | tail -n -1); && \ + tar -xf $GZ; && \ + DIR=$(echo $GZ | sed -e "s/.tar.gz$//"); && \ + cp -R ${DIR}/bin $LK_HOME; && \ + cp -R ${DIR}/modules $LK_HOME; && \ + cp -R ${DIR}/labkeywebapp $LK_HOME; && \ + cp -R ${DIR}/pipeline-lib $LK_HOME; && \ + mkdir -p /labkey/apps/tomcat/lib/; && \ + cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/; && \ + rm -rf ${DIR}) From b084e0ed4377eb15cafecd75f544647d3cf787dd Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 3 May 2016 11:42:52 -0700 Subject: [PATCH 3/9] remove semicolons --- docker/base/Dockerfile | 8 ++++---- docker/remote/Dockerfile | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index fa60794..21ba0d8 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -22,15 +22,15 @@ RUN mkdir -p /labkey/svn # Do all updates and install java in single step to slim down image # Sequence tools are installed using a separate shell script since this config is shared by non-docker usages -RUN (echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ - apt-get update; \ - apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common oracle-java8-installer; \ + apt-get update \ + apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common oracle-java8-installer \ rm -rf /var/cache/oracle-jdk8-installer && \ svn co --username cpas --password cpas --no-auth-cache https://hedgehog.fhcrc.org/tor/stedi/branches/${SVN_BRANCH}/externalModules/labModules/SequenceAnalysis/pipeline_code ${LK_HOME}/svn/ && \ ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -c && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ - apt-get clean -y) + apt-get clean -y ENV JAVA_HOME /usr/lib/jvm/java-8-oracle ENV PATH $LK_HOME/bin:$JAVA_HOME/bin:$PATH diff --git a/docker/remote/Dockerfile b/docker/remote/Dockerfile index 34a2250..6efe42b 100644 --- a/docker/remote/Dockerfile +++ b/docker/remote/Dockerfile @@ -34,13 +34,13 @@ run mkdir -p /labkey/configs RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/${GZ_PREFIX}-{build.number}-${DIST_NAME}-bin.tar.gz && \ mv ./teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/*.gz ./ && \ rm -Rf ./teamcity.labkey.org && \ - GZ=$(ls -tr | grep '^LabKey.*\.gz$' | tail -n -1); && \ - tar -xf $GZ; && \ - DIR=$(echo $GZ | sed -e "s/.tar.gz$//"); && \ - cp -R ${DIR}/bin $LK_HOME; && \ - cp -R ${DIR}/modules $LK_HOME; && \ - cp -R ${DIR}/labkeywebapp $LK_HOME; && \ - cp -R ${DIR}/pipeline-lib $LK_HOME; && \ - mkdir -p /labkey/apps/tomcat/lib/; && \ - cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/; && \ - rm -rf ${DIR}) + GZ=$(ls -tr | grep '^LabKey.*\.gz$' | tail -n -1) && \ + tar -xf $GZ && \ + DIR=$(echo $GZ | sed -e "s/.tar.gz$//") && \ + cp -R ${DIR}/bin $LK_HOME && \ + cp -R ${DIR}/modules $LK_HOME && \ + cp -R ${DIR}/labkeywebapp $LK_HOME && \ + cp -R ${DIR}/pipeline-lib $LK_HOME && \ + mkdir -p /labkey/apps/tomcat/lib/ && \ + cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/ && \ + rm -rf ${DIR} From e48e13aa39d3e032fc49740493aab309d3db60ff Mon Sep 17 00:00:00 2001 From: bbimber Date: Tue, 3 May 2016 12:34:15 -0700 Subject: [PATCH 4/9] fix typo --- docker/base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 21ba0d8..c141c59 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -24,8 +24,8 @@ RUN mkdir -p /labkey/svn # Sequence tools are installed using a separate shell script since this config is shared by non-docker usages RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ add-apt-repository -y ppa:webupd8team/java && \ - apt-get update \ - apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common oracle-java8-installer \ + apt-get update && \ + apt-get -y -q install wget tar unzip subversion wget python-software-properties software-properties-common oracle-java8-installer && \ rm -rf /var/cache/oracle-jdk8-installer && \ svn co --username cpas --password cpas --no-auth-cache https://hedgehog.fhcrc.org/tor/stedi/branches/${SVN_BRANCH}/externalModules/labModules/SequenceAnalysis/pipeline_code ${LK_HOME}/svn/ && \ ${LK_HOME}/svn/sequence_tools_install.sh -d ${LK_HOME} -c && \ From 1c99389e6915676a3910303418e1212a7124954d Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 May 2016 08:18:21 -0700 Subject: [PATCH 5/9] attempt to explode modules during docker build to make local deployment quicker --- docker/remote/Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/remote/Dockerfile b/docker/remote/Dockerfile index 6efe42b..cca5174 100644 --- a/docker/remote/Dockerfile +++ b/docker/remote/Dockerfile @@ -4,7 +4,7 @@ # sequence tools used by the pipelines # -from bbimber/discvr-seq +from bbimber/discvr-seq:16.1 maintainer bbimber@gmail.com # these will vary by LabKey version: @@ -27,10 +27,10 @@ ENV LK_HOME /labkey # Consider adding --add-host activemq:XXX.XXX.XXX.XXX, depending on how pipelineConfig.xml is set up # -run mkdir -p /labkey/ -run mkdir -p /labkey/configs +run mkdir -p /labkey/ && mkdir -p /labkey/configs && mkdir -p /labkey/logs # download latest build +# Note: the final line is intended to force the modules/webapp dirs to be exploded RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/${GZ_PREFIX}-{build.number}-${DIST_NAME}-bin.tar.gz && \ mv ./teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/*.gz ./ && \ rm -Rf ./teamcity.labkey.org && \ @@ -41,6 +41,11 @@ RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.o cp -R ${DIR}/modules $LK_HOME && \ cp -R ${DIR}/labkeywebapp $LK_HOME && \ cp -R ${DIR}/pipeline-lib $LK_HOME && \ + cp -R ${DIR}/labkeyBootstrap.jar $LK_HOME && \ mkdir -p /labkey/apps/tomcat/lib/ && \ cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/ && \ - rm -rf ${DIR} + cd $LK_HOME && \ + rm -rf ${DIR} && \ + rm -Rf $GZ && \ + java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap + From 12d0dd3ec1ac5a0c2f2ccf853f68a29d2ff68298 Mon Sep 17 00:00:00 2001 From: bbimber Date: Fri, 6 May 2016 09:26:10 -0700 Subject: [PATCH 6/9] update labkeyBootstrap.jar location --- docker/remote/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/remote/Dockerfile b/docker/remote/Dockerfile index cca5174..fdbd1bb 100644 --- a/docker/remote/Dockerfile +++ b/docker/remote/Dockerfile @@ -41,7 +41,7 @@ RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.o cp -R ${DIR}/modules $LK_HOME && \ cp -R ${DIR}/labkeywebapp $LK_HOME && \ cp -R ${DIR}/pipeline-lib $LK_HOME && \ - cp -R ${DIR}/labkeyBootstrap.jar $LK_HOME && \ + cp -R ${DIR}/tomcat-lib/labkeyBootstrap.jar $LK_HOME && \ mkdir -p /labkey/apps/tomcat/lib/ && \ cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/ && \ cd $LK_HOME && \ From 2fe1c00a5167a6a8c7c658824571ccb641fb6d40 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 8 May 2016 16:29:03 -0700 Subject: [PATCH 7/9] split dockerfile to get around dockerhub build timeout --- docker/exacloud/Dockerfile | 12 ++++++++++++ docker/remote/Dockerfile | 5 ++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 docker/exacloud/Dockerfile diff --git a/docker/exacloud/Dockerfile b/docker/exacloud/Dockerfile new file mode 100644 index 0000000..e9e37b5 --- /dev/null +++ b/docker/exacloud/Dockerfile @@ -0,0 +1,12 @@ +# DOCKER-VERSION 1.0.0 +# +# Dockerfile for remote pipeline DISCVR-Seq Servers. This will install remote LabKey Server code in addition to various +# sequence tools used by the pipelines +# + +from bbimber/discvr-seq:16.1server +maintainer bbimber@gmail.com + +# the only reason for this command is to pre-explode all the modules to save time when running the container +RUN java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap + diff --git a/docker/remote/Dockerfile b/docker/remote/Dockerfile index fdbd1bb..39def3e 100644 --- a/docker/remote/Dockerfile +++ b/docker/remote/Dockerfile @@ -1,4 +1,5 @@ # DOCKER-VERSION 1.0.0 +# DOCKER-VERSION 1.0.0 # # Dockerfile for remote pipeline DISCVR-Seq Servers. This will install remote LabKey Server code in addition to various # sequence tools used by the pipelines @@ -30,7 +31,6 @@ ENV LK_HOME /labkey run mkdir -p /labkey/ && mkdir -p /labkey/configs && mkdir -p /labkey/logs # download latest build -# Note: the final line is intended to force the modules/webapp dirs to be exploded RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/${GZ_PREFIX}-{build.number}-${DIST_NAME}-bin.tar.gz && \ mv ./teamcity.labkey.org/guestAuth/repository/download/${TC_PROJECT}/.lastSuccessful/${DIST_NAME}/*.gz ./ && \ rm -Rf ./teamcity.labkey.org && \ @@ -46,6 +46,5 @@ RUN wget -r --trust-server-names --no-check-certificate http://teamcity.labkey.o cp -f ${DIR}/tomcat-lib/*.jar /labkey/apps/tomcat/lib/ && \ cd $LK_HOME && \ rm -rf ${DIR} && \ - rm -Rf $GZ && \ - java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap + rm -Rf $GZ From 722390c044d05bd41c09ac44ea0f967b6513d502 Mon Sep 17 00:00:00 2001 From: bbimber Date: Sun, 8 May 2016 21:04:48 -0700 Subject: [PATCH 8/9] fix path --- docker/exacloud/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/exacloud/Dockerfile b/docker/exacloud/Dockerfile index e9e37b5..46a5882 100644 --- a/docker/exacloud/Dockerfile +++ b/docker/exacloud/Dockerfile @@ -8,5 +8,5 @@ from bbimber/discvr-seq:16.1server maintainer bbimber@gmail.com # the only reason for this command is to pre-explode all the modules to save time when running the container -RUN java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap +RUN cd $LK_HOME && java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap From 98b8d5abf4c0c1d11ef54b0b59053da30b0c5870 Mon Sep 17 00:00:00 2001 From: bbimber Date: Mon, 9 May 2016 21:55:57 -0700 Subject: [PATCH 9/9] debug dockerhub build --- docker/exacloud/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/exacloud/Dockerfile b/docker/exacloud/Dockerfile index 46a5882..cdaa3b0 100644 --- a/docker/exacloud/Dockerfile +++ b/docker/exacloud/Dockerfile @@ -8,5 +8,5 @@ from bbimber/discvr-seq:16.1server maintainer bbimber@gmail.com # the only reason for this command is to pre-explode all the modules to save time when running the container -RUN cd $LK_HOME && java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap +RUN date && cd $LK_HOME && java -cp $JAVA_HOME/lib/tools.jar:$LK_HOME/labkeyBootstrap.jar org.labkey.bootstrap.ClusterBootstrap