From 5c36e8d58e3c8bcd0ee0fca16d358ca950b0cabc Mon Sep 17 00:00:00 2001 From: Nathan Dunn Date: Tue, 19 Nov 2019 19:29:34 -0800 Subject: [PATCH] Fix 2307 v1 (#2314) * add a todo * more awkward pirouettes... * solve chicken and egg problem * try (just a little bit harder) * try to reduce docker image size * try to implement #2307 * fixes #2307 * fix * fix * making sequence no longer read-only * fixed the order of the organism being fixed * reversed the reloading flag * add env var to enable more debug in docker mode * minor updates * fixed the multipart form check --- .dockerignore | 3 + .travis.yml | 4 + Dockerfile | 45 +++----- docker-files/docker-apollo-config.groovy | 7 ++ .../bbop/apollo/AnnotatorController.groovy | 2 +- .../org/bbop/apollo/OrganismController.groovy | 15 ++- .../domain/org/bbop/apollo/Sequence.groovy | 1 - .../org/bbop/apollo/SequenceService.groovy | 103 ++++++++++++++---- .../sequence/search/blat/BlatCommandLine.java | 2 +- 9 files changed, 123 insertions(+), 59 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..5fedadb675 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.gitignore +.github diff --git a/.travis.yml b/.travis.yml index 9443f42cb2..ffabef6174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,3 +80,7 @@ script: python setup.py nosetests killall java || true fi + +jobs: + allow_failures: + - env: DB=h2 TEST_SUITE=python-apollo diff --git a/Dockerfile b/Dockerfile index 91068e516a..be686ce4bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ -# WebApollo -# VERSION 2.1.X +# Apollo2 FROM tomcat:9-jdk8 MAINTAINER Nathan Dunn ENV DEBIAN_FRONTEND noninteractive +ENV CATALINA_HOME=/usr/local/tomcat +ENV CONTEXT_PATH ROOT + RUN apt-get -qq update --fix-missing && \ apt-get --no-install-recommends -y install \ git build-essential maven libpq-dev postgresql-common openjdk-8-jdk wget \ @@ -20,51 +22,34 @@ RUN npm i -g yarn RUN cp /usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/tools.jar && \ useradd -ms /bin/bash -d /apollo apollo -ENV WEBAPOLLO_VERSION develop -#RUN ls -la /apollo/ +RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat && \ + chmod +x /usr/local/bin/blat && \ + curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit && \ + chmod +x /usr/local/bin/faToTwoBit && \ + wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins26/chado-1.31.sql.gz -O /chado.sql.gz && \ + gunzip /chado.sql.gz + #NOTE, we had problems with the build the archive-file coming in from github so using a clone instead -#RUN curl -L https://github.com/GMOD/Apollo/archive/${WEBAPOLLO_VERSION}.tar.gz | tar xzf - --strip-components=1 -C /apollo -# RUN git clone --depth 1 --single-branch --branch ${WEBAPOLLO_VERSION} https://github.com/gmod/apollo /apollo/apollo-clone -# RUN mv /apollo/apollo-clone/* /apollo && rm -rf /apollo/apollo-clone ADD . /apollo -RUN rm -rf .git .gitignore -#COPY * /apollo # install grails COPY docker-files/build.sh /bin/build.sh ADD docker-files/docker-apollo-config.groovy /apollo/apollo-config.groovy RUN chown -R apollo:apollo /apollo -RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat -RUN chmod +x /usr/local/bin/blat -RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit -RUN chmod +x /usr/local/bin/faToTwoBit - USER apollo -RUN curl -s get.sdkman.io | bash -RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" -RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" - - -RUN /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh" +RUN curl -s get.sdkman.io | bash && \ + /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" && \ + /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" && \ + /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh" USER root -ENV CATALINA_HOME=/usr/local/tomcat RUN rm -rf ${CATALINA_HOME}/webapps/* && \ cp /apollo/apollo*.war ${CATALINA_HOME}/apollo.war -ENV CONTEXT_PATH ROOT - -# Download chado schema -RUN wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins97/chado-1.31.sql.gz -O /chado.sql.gz && \ - gunzip /chado.sql.gz - - ADD docker-files/createenv.sh /createenv.sh CMD "/createenv.sh" ADD docker-files/launch.sh /launch.sh CMD "/launch.sh" - - diff --git a/docker-files/docker-apollo-config.groovy b/docker-files/docker-apollo-config.groovy index bf681cde37..6def5d52c8 100644 --- a/docker-files/docker-apollo-config.groovy +++ b/docker-files/docker-apollo-config.groovy @@ -77,6 +77,13 @@ environments { } } +if (System.getenv("WEBAPOLLO_DEBUG") == "true") { + log4j.main = { + debug "grails.app" + } +} + + apollo { common_data_directory = System.getenv("WEBAPOLLO_COMMON_DATA") ? System.getenv("WEBAPOLLO_COMMON_DATA") : "/data/temporary/apollo_data" default_minimum_intron_size = System.getenv("WEBAPOLLO_MINIMUM_INTRON_SIZE") ? System.getenv("WEBAPOLLO_MINIMUM_INTRON_SIZE").toInteger() : 1 diff --git a/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy b/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy index ae46b1f7f1..0d1dee2f15 100644 --- a/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy +++ b/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy @@ -567,7 +567,7 @@ class AnnotatorController { * This is a public passthrough to version */ def version() { - println "versoin " + println "version " } def about(){ diff --git a/grails-app/controllers/org/bbop/apollo/OrganismController.groovy b/grails-app/controllers/org/bbop/apollo/OrganismController.groovy index d96ec58b1a..9868c12b64 100644 --- a/grails-app/controllers/org/bbop/apollo/OrganismController.groovy +++ b/grails-app/controllers/org/bbop/apollo/OrganismController.groovy @@ -22,6 +22,7 @@ import org.restapidoc.annotation.RestApiParams import org.restapidoc.pojo.RestApiParamType import org.restapidoc.pojo.RestApiVerb import org.springframework.web.multipart.commons.CommonsMultipartFile +import org.springframework.web.multipart.support.AbstractMultipartHttpServletRequest import javax.servlet.http.HttpServletResponse import java.nio.file.FileSystems @@ -101,7 +102,7 @@ class OrganismController { assert directoryToRemove.deleteDir() } - render findAllOrganisms() + findAllOrganisms() } catch (Exception e) { @@ -1204,6 +1205,7 @@ class OrganismController { def c = Sequence.createCriteria() sequenceList = c.list { eq('organism', organism) + order('name',"asc") } log.debug "Sequence list fetched at getSequencesForOrganism: ${sequenceList}" } else { @@ -1289,13 +1291,14 @@ class OrganismController { madeObsolete = !organism.obsolete && (organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false) organism.obsolete = organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false organism.nonDefaultTranslationTable = organismJson.nonDefaultTranslationTable ?: organism.nonDefaultTranslationTable - if (organism.genomeFasta) { + if(organism.genomeFasta) { // update location of genome fasta sequenceService.updateGenomeFasta(organism) } +// CommonsMultipartFile organismDataFile = request.getFile(FeatureStringEnum.ORGANISM_DATA.value) CommonsMultipartFile organismDataFile = null - if (!request instanceof ShiroHttpServletRequest) { + if (request instanceof AbstractMultipartHttpServletRequest) { organismDataFile = request.getFile(FeatureStringEnum.ORGANISM_DATA.value) } String foundBlatdb = null @@ -1325,7 +1328,7 @@ class OrganismController { } organism.save(flush: true, insert: false, failOnError: true) - if (oldOrganismDirectory!=organism.directory && !noReloadSequencesIfOrganismChanges) { + if ((organismDataFile || oldOrganismDirectory!=organism.directory) && !noReloadSequencesIfOrganismChanges) { // we need to reload sequenceService.loadRefSeqs(organism) } @@ -1337,7 +1340,7 @@ class OrganismController { } else { throw new Exception('organism not found') } - render findAllOrganisms() as JSON + findAllOrganisms() } catch (e) { def error = [error: 'problem saving organism: ' + e] @@ -1422,7 +1425,7 @@ class OrganismController { organism = Organism.findByCommonName(requestObject.organism) if (!organism) organism = Organism.findById(requestObject.organism) } catch (e) { - log.error("Unable to find organism for ${requestObject.organism}") + log.warn("Unable to find organism for ${requestObject.organism}") organism = null } if (!organism) { diff --git a/grails-app/domain/org/bbop/apollo/Sequence.groovy b/grails-app/domain/org/bbop/apollo/Sequence.groovy index 65acc582ab..a9e8b3a905 100644 --- a/grails-app/domain/org/bbop/apollo/Sequence.groovy +++ b/grails-app/domain/org/bbop/apollo/Sequence.groovy @@ -20,7 +20,6 @@ class Sequence { ] static mapping = { - cache usage: 'read-only' end column: 'sequence_end' start column: 'sequence_start' featureLocations cascade: 'all-delete-orphan' diff --git a/grails-app/services/org/bbop/apollo/SequenceService.groovy b/grails-app/services/org/bbop/apollo/SequenceService.groovy index 24d420930a..ff9c316a53 100644 --- a/grails-app/services/org/bbop/apollo/SequenceService.groovy +++ b/grails-app/services/org/bbop/apollo/SequenceService.groovy @@ -197,7 +197,7 @@ class SequenceService { } currentOffset -= alterationLength; } - // Substitions + // Substitutions else if (sequenceAlteration.instanceOf == SubstitutionArtifact.canonicalName) { int start = strand == Strand.NEGATIVE ? localCoordinate - (alterationLength - 1) : localCoordinate; residues.replace(start + currentOffset, @@ -325,9 +325,16 @@ class SequenceService { } def sequences = Sequence.findAllByOrganism(organism) - def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s join s.organism o where o = :organism",[organism:organism]) - Preference.deleteAll(preferences) - Sequence.deleteAll(sequences) + def seqsMap = [:] + sequences.each { sequence -> + seqsMap[sequence.name] = sequence.length + } + +// def knownSequences = Sequence.findAllByOrganism(organism) +// def seqsMap = [:] +// knownSequences.each { sequence -> +// seqsMap[sequence.name] = sequence.length +// } // this will fail if folks have actively been working on this and preferences are set // otherwise if we remove all of the sequences annotations will need to be removed as well @@ -341,14 +348,41 @@ class SequenceService { //workaround for jbrowse refSeqs that have no length element length = refSeq.end - refSeq.start } - Sequence sequence = new Sequence( - organism: organism - , length: length - , seqChunkSize: refSeq.seqChunkSize - , start: refSeq.start - , end: refSeq.end - , name: refSeq.name - ).save(failOnError: true) + if (!seqsMap.containsKey(refSeq.name)) { + Sequence sequence = new Sequence( + organism: organism + , length: length + , seqChunkSize: refSeq.seqChunkSize + , start: refSeq.start + , end: refSeq.end + , name: refSeq.name + ).save(failOnError: true) + log.debug "added sequence ${sequence}" + } + else if (seqsMap[refSeq.name] != length) { + Sequence sequence = Sequence.findByNameAndOrganism(refSeq.name,organism) + sequence.length = length + sequence.seqChunkSize = refSeq.seqChunkSize + sequence.start = refSeq.start + sequence.end = refSeq.end +// sequence.name = refSeq.name + sequence.save(failOnError: true,insert:false) +// def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s where s = :sequence",[sequence:seqsMap[refSeq.name]]) +// Preference.deleteAll(preferences) +// Sequence.delete(seqsMap[refSeq.name]) +// Sequence sequence = new Sequence( +// organism: organism +// , length: length +// , seqChunkSize: refSeq.seqChunkSize +// , start: refSeq.start +// , end: refSeq.end +// , name: refSeq.name +// ).save(failOnError: true) + log.debug "uddated sequence ${sequence}" + } + else { + log.debug "skipped existing unchanged sequence ${refSeq.name}" + } } organism.valid = true @@ -372,18 +406,47 @@ class SequenceService { FastaSequenceIndex index = new FastaSequenceIndex(genomeFastaIndexFile) log.info "an indexed fasta ${index}" log.info "an indexed fasta size ${index.size()}" + def knownSequences = Sequence.findAllByOrganism(organism) + def seqsMap = [:] + knownSequences.each { sequence -> + seqsMap[sequence.name] = sequence.length + } // reading the index def iterator = index.iterator() while (iterator.hasNext()) { def entry = iterator.next() - Sequence sequence = new Sequence( - organism: organism, - length: entry.size, - start: 0, - end: entry.size, - name: entry.contig - ).save(failOnError: true) - log.debug "added sequence ${sequence}" + if (!seqsMap.containsKey(entry.contig)) { +// if (!seqsMap.containsKey(entry.contig) || seqsMap[entry.contig] != entry.size) { + Sequence sequence = new Sequence( + organism: organism, + length: entry.size, + start: 0, + end: entry.size, + name: entry.contig + ).save(failOnError: true,insert: true) + log.debug "added sequence ${sequence}" + } + else if (seqsMap[entry.contig] != entry.size) { +// def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s where s = :sequence",[sequence:sequence]) +// Preference.deleteAll(preferences) +// Sequence.delete(seqsMap[entry.contig]) +// Sequence sequence = new Sequence( +// organism: organism, +// length: entry.size, +// start: 0, +// end: entry.size, +// name: entry.contig +// ).save(failOnError: true) + Sequence sequence = Sequence.findByNameAndOrganism(entry.contig,organism) + sequence.length = entry.size as Integer + sequence.start = 0 + sequence.end = entry.size as Integer + sequence.save(failOnError: true,insert:false) + log.debug "replaced sequence ${sequence}" + } + else { + log.debug "skipped existing unchanged sequence ${entry.contig}" + } } organism.valid = true diff --git a/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java b/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java index a13e70da81..9fcafff3cf 100644 --- a/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java +++ b/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java @@ -52,7 +52,7 @@ public Collection search(String uniqueToken, String query, Strin p = Files.createTempDirectory(new File(tmpDir).toPath(),"blat_tmp"); } dir = p.toFile(); - + return runSearch(dir, query, databaseId); } catch (IOException e) {