Fix 2307 v1 (#2314)

* add a todo * more awkward pirouettes... * solve chicken and egg problem * try (just a little bit harder) * try to reduce docker image size * try to implement #2307 * fixes #2307 * fix * fix * making sequence no longer read-only * fixed the order of the organism being fixed * reversed the reloading flag * add env var to enable more debug in docker mode * minor updates * fixed the multipart form check
GMOD · Nov 20, 2019 · 5c36e8d · 5c36e8d
1 parent 1f90932
commit 5c36e8d
Show file tree

Hide file tree

Showing 9 changed files with 123 additions and 59 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+.git
+.gitignore
+.github
diff --git a/.travis.yml b/.travis.yml
@@ -80,3 +80,7 @@ script:
       python setup.py nosetests
       killall java || true
     fi
+
+jobs:
+  allow_failures:
+    - env: DB=h2 TEST_SUITE=python-apollo
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,11 @@
-# WebApollo
-# VERSION 2.1.X
+# Apollo2
 FROM tomcat:9-jdk8
 MAINTAINER Nathan Dunn <nathandunn@lbl.gov>
 ENV DEBIAN_FRONTEND noninteractive
 
+ENV CATALINA_HOME=/usr/local/tomcat
+ENV CONTEXT_PATH ROOT
+
 RUN apt-get -qq update --fix-missing && \
 	apt-get --no-install-recommends -y install \
 	git build-essential maven libpq-dev postgresql-common openjdk-8-jdk wget \
@@ -20,51 +22,34 @@ RUN npm i -g yarn
 RUN cp /usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/tools.jar && \
 	useradd -ms /bin/bash -d /apollo apollo
 
-ENV WEBAPOLLO_VERSION develop
-#RUN ls -la /apollo/
+RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat && \
+ 		chmod +x /usr/local/bin/blat && \
+ 		curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit && \
+ 		chmod +x /usr/local/bin/faToTwoBit && \
+		wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins26/chado-1.31.sql.gz -O /chado.sql.gz && \
+		gunzip /chado.sql.gz
+
 #NOTE, we had problems with the build the archive-file coming in from github so using a clone instead
-#RUN curl -L https://github.com/GMOD/Apollo/archive/${WEBAPOLLO_VERSION}.tar.gz | tar xzf - --strip-components=1 -C /apollo
-# RUN git clone --depth 1 --single-branch --branch ${WEBAPOLLO_VERSION} https://github.com/gmod/apollo /apollo/apollo-clone
-# RUN mv /apollo/apollo-clone/* /apollo && rm -rf /apollo/apollo-clone
 ADD . /apollo
-RUN rm -rf .git .gitignore
-#COPY * /apollo
 
 # install grails
 COPY docker-files/build.sh /bin/build.sh
 ADD docker-files/docker-apollo-config.groovy /apollo/apollo-config.groovy
 
 RUN chown -R apollo:apollo /apollo
-RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat
-RUN chmod +x /usr/local/bin/blat
-RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit
-RUN chmod +x /usr/local/bin/faToTwoBit
-
 
 USER apollo
-RUN curl -s get.sdkman.io | bash
-RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5"
-RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1"
-
-
-RUN /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh"
+RUN curl -s get.sdkman.io | bash && \
+		/bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" && \
+ 		/bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" && \
+ 		/bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh"
 
 USER root
-ENV CATALINA_HOME=/usr/local/tomcat
 RUN rm -rf ${CATALINA_HOME}/webapps/* && \
 	cp /apollo/apollo*.war ${CATALINA_HOME}/apollo.war
 
-ENV CONTEXT_PATH ROOT
-
-# Download chado schema
-RUN wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins97/chado-1.31.sql.gz -O /chado.sql.gz && \
-	gunzip /chado.sql.gz
-
-
 ADD docker-files/createenv.sh /createenv.sh
 CMD "/createenv.sh"
 
 ADD docker-files/launch.sh /launch.sh
 CMD "/launch.sh"
-
-
diff --git a/docker-files/docker-apollo-config.groovy b/docker-files/docker-apollo-config.groovy
@@ -77,6 +77,13 @@ environments {
     }
 }
 
+if (System.getenv("WEBAPOLLO_DEBUG") == "true") {
+    log4j.main = {
+        debug "grails.app"
+    }
+}
+
+
 apollo {
     common_data_directory = System.getenv("WEBAPOLLO_COMMON_DATA") ? System.getenv("WEBAPOLLO_COMMON_DATA") : "/data/temporary/apollo_data"
     default_minimum_intron_size = System.getenv("WEBAPOLLO_MINIMUM_INTRON_SIZE") ? System.getenv("WEBAPOLLO_MINIMUM_INTRON_SIZE").toInteger() : 1

diff --git a/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy b/grails-app/controllers/org/bbop/apollo/AnnotatorController.groovy
@@ -567,7 +567,7 @@ class AnnotatorController {
  * This is a public passthrough to version
  */
     def version() {
-      println "versoin "
+      println "version "
     }
 
   def about(){

diff --git a/grails-app/controllers/org/bbop/apollo/OrganismController.groovy b/grails-app/controllers/org/bbop/apollo/OrganismController.groovy
@@ -22,6 +22,7 @@ import org.restapidoc.annotation.RestApiParams
 import org.restapidoc.pojo.RestApiParamType
 import org.restapidoc.pojo.RestApiVerb
 import org.springframework.web.multipart.commons.CommonsMultipartFile
+import org.springframework.web.multipart.support.AbstractMultipartHttpServletRequest
 
 import javax.servlet.http.HttpServletResponse
 import java.nio.file.FileSystems
@@ -101,7 +102,7 @@ class OrganismController {
         assert directoryToRemove.deleteDir()
       }
 
-      render findAllOrganisms()
+      findAllOrganisms()
 
     }
     catch (Exception e) {
@@ -1204,6 +1205,7 @@ class OrganismController {
       def c = Sequence.createCriteria()
       sequenceList = c.list {
         eq('organism', organism)
+        order('name',"asc")
       }
       log.debug "Sequence list fetched at getSequencesForOrganism: ${sequenceList}"
     } else {
@@ -1289,13 +1291,14 @@ class OrganismController {
         madeObsolete = !organism.obsolete && (organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false)
         organism.obsolete = organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false
         organism.nonDefaultTranslationTable = organismJson.nonDefaultTranslationTable ?: organism.nonDefaultTranslationTable
-        if (organism.genomeFasta) {
+        if(organism.genomeFasta) {
           // update location of genome fasta
           sequenceService.updateGenomeFasta(organism)
         }
 
+//        CommonsMultipartFile organismDataFile = request.getFile(FeatureStringEnum.ORGANISM_DATA.value)
         CommonsMultipartFile organismDataFile = null
-        if (!request instanceof ShiroHttpServletRequest) {
+        if (request instanceof AbstractMultipartHttpServletRequest) {
           organismDataFile = request.getFile(FeatureStringEnum.ORGANISM_DATA.value)
         }
         String foundBlatdb = null
@@ -1325,7 +1328,7 @@ class OrganismController {
           }
           organism.save(flush: true, insert: false, failOnError: true)
 
-          if (oldOrganismDirectory!=organism.directory && !noReloadSequencesIfOrganismChanges) {
+          if ((organismDataFile || oldOrganismDirectory!=organism.directory) && !noReloadSequencesIfOrganismChanges) {
             // we need to reload
             sequenceService.loadRefSeqs(organism)
           }
@@ -1337,7 +1340,7 @@ class OrganismController {
       } else {
         throw new Exception('organism not found')
       }
-      render findAllOrganisms() as JSON
+      findAllOrganisms()
     }
     catch (e) {
       def error = [error: 'problem saving organism: ' + e]
@@ -1422,7 +1425,7 @@ class OrganismController {
           organism = Organism.findByCommonName(requestObject.organism)
           if (!organism) organism = Organism.findById(requestObject.organism)
         } catch (e) {
-          log.error("Unable to find organism for ${requestObject.organism}")
+          log.warn("Unable to find organism for ${requestObject.organism}")
           organism = null
         }
         if (!organism) {

diff --git a/grails-app/domain/org/bbop/apollo/Sequence.groovy b/grails-app/domain/org/bbop/apollo/Sequence.groovy
@@ -20,7 +20,6 @@ class Sequence {
     ]
 
     static mapping = {
-        cache usage: 'read-only'
         end column: 'sequence_end'
         start column: 'sequence_start'
         featureLocations cascade: 'all-delete-orphan'

diff --git a/grails-app/services/org/bbop/apollo/SequenceService.groovy b/grails-app/services/org/bbop/apollo/SequenceService.groovy
@@ -197,7 +197,7 @@ class SequenceService {
                 }
                 currentOffset -= alterationLength;
             }
-            // Substitions
+            // Substitutions
             else if (sequenceAlteration.instanceOf == SubstitutionArtifact.canonicalName) {
                 int start = strand == Strand.NEGATIVE ? localCoordinate - (alterationLength - 1) : localCoordinate;
                 residues.replace(start + currentOffset,
@@ -325,9 +325,16 @@ class SequenceService {
             }
 
           def sequences = Sequence.findAllByOrganism(organism)
-          def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s join s.organism o where o = :organism",[organism:organism])
-          Preference.deleteAll(preferences)
-          Sequence.deleteAll(sequences)
+          def seqsMap = [:]
+          sequences.each { sequence ->
+              seqsMap[sequence.name] = sequence.length
+          }
+
+//          def knownSequences = Sequence.findAllByOrganism(organism)
+//          def seqsMap = [:]
+//          knownSequences.each { sequence ->
+//              seqsMap[sequence.name] = sequence.length
+//          }
 
           // this will fail if folks have actively been working on this and preferences are set
           // otherwise if we remove all of the sequences annotations will need to be removed as well
@@ -341,14 +348,41 @@ class SequenceService {
                     //workaround for jbrowse refSeqs that have no length element
                     length = refSeq.end - refSeq.start
                 }
-                Sequence sequence = new Sequence(
-                        organism: organism
-                        , length: length
-                        , seqChunkSize: refSeq.seqChunkSize
-                        , start: refSeq.start
-                        , end: refSeq.end
-                        , name: refSeq.name
-                ).save(failOnError: true)
+                if (!seqsMap.containsKey(refSeq.name)) {
+                    Sequence sequence = new Sequence(
+                            organism: organism
+                            , length: length
+                            , seqChunkSize: refSeq.seqChunkSize
+                            , start: refSeq.start
+                            , end: refSeq.end
+                            , name: refSeq.name
+                    ).save(failOnError: true)
+                    log.debug "added sequence ${sequence}"
+                }
+                else if (seqsMap[refSeq.name] != length) {
+                  Sequence sequence = Sequence.findByNameAndOrganism(refSeq.name,organism)
+                  sequence.length = length
+                  sequence.seqChunkSize = refSeq.seqChunkSize
+                  sequence.start = refSeq.start
+                  sequence.end = refSeq.end
+//                  sequence.name = refSeq.name
+                  sequence.save(failOnError: true,insert:false)
+//                    def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s where s = :sequence",[sequence:seqsMap[refSeq.name]])
+//                    Preference.deleteAll(preferences)
+//                    Sequence.delete(seqsMap[refSeq.name])
+//                    Sequence sequence = new Sequence(
+//                            organism: organism
+//                            , length: length
+//                            , seqChunkSize: refSeq.seqChunkSize
+//                            , start: refSeq.start
+//                            , end: refSeq.end
+//                            , name: refSeq.name
+//                    ).save(failOnError: true)
+                    log.debug "uddated sequence ${sequence}"
+                }
+                else {
+                    log.debug "skipped existing unchanged sequence ${refSeq.name}"
+                }
             }
 
             organism.valid = true
@@ -372,18 +406,47 @@ class SequenceService {
                 FastaSequenceIndex index = new FastaSequenceIndex(genomeFastaIndexFile)
                 log.info "an indexed fasta ${index}"
                 log.info "an indexed fasta size ${index.size()}"
+                def knownSequences = Sequence.findAllByOrganism(organism)
+                def seqsMap = [:]
+                knownSequences.each { sequence ->
+                    seqsMap[sequence.name] = sequence.length
+                }
                 // reading the index
                 def iterator = index.iterator()
                 while (iterator.hasNext()) {
                     def entry = iterator.next()
-                    Sequence sequence = new Sequence(
-                            organism: organism,
-                            length: entry.size,
-                            start: 0,
-                            end: entry.size,
-                            name: entry.contig
-                    ).save(failOnError: true)
-                    log.debug "added sequence ${sequence}"
+                  if (!seqsMap.containsKey(entry.contig)) {
+//                    if (!seqsMap.containsKey(entry.contig) || seqsMap[entry.contig] != entry.size) {
+                        Sequence sequence = new Sequence(
+                                organism: organism,
+                                length: entry.size,
+                                start: 0,
+                                end: entry.size,
+                                name: entry.contig
+                        ).save(failOnError: true,insert: true)
+                        log.debug "added sequence ${sequence}"
+                    }
+                    else if (seqsMap[entry.contig] != entry.size) {
+//                      def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s where s = :sequence",[sequence:sequence])
+//                      Preference.deleteAll(preferences)
+//                      Sequence.delete(seqsMap[entry.contig])
+//                      Sequence sequence = new Sequence(
+//                        organism: organism,
+//                        length: entry.size,
+//                        start: 0,
+//                        end: entry.size,
+//                        name: entry.contig
+//                      ).save(failOnError: true)
+                      Sequence sequence = Sequence.findByNameAndOrganism(entry.contig,organism)
+                      sequence.length = entry.size as Integer
+                      sequence.start = 0
+                      sequence.end = entry.size as Integer
+                      sequence.save(failOnError: true,insert:false)
+                    log.debug "replaced sequence ${sequence}"
+                    }
+                    else {
+                        log.debug "skipped existing unchanged sequence ${entry.contig}"
+                    }
                 }
 
                 organism.valid = true

diff --git a/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java b/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java
@@ -52,7 +52,7 @@ public Collection<BlastAlignment> search(String uniqueToken, String query, Strin
               p = Files.createTempDirectory(new File(tmpDir).toPath(),"blat_tmp");
           }
           dir = p.toFile();
-          
+
           return runSearch(dir, query, databaseId);
         }
         catch (IOException e) {