Merge 8e43ef4 into f6a3eeb

.dockerignore

@@ -0,0 +1,3 @@
+.git
+.gitignore
+.github

.travis.yml

@@ -80,3 +80,7 @@ script:
       python setup.py nosetests
       killall java || true
     fi
+
+jobs:
+  allow_failures:
+    - env: DB=h2 TEST_SUITE=python-apollo

Dockerfile

@@ -1,9 +1,11 @@
-# WebApollo
-# VERSION 2.1.X
+# Apollo2
 FROM tomcat:9-jdk8
 MAINTAINER Nathan Dunn <nathandunn@lbl.gov>
 ENV DEBIAN_FRONTEND noninteractive
 
+ENV CATALINA_HOME=/usr/local/tomcat
+ENV CONTEXT_PATH ROOT
+
 RUN apt-get -qq update --fix-missing && \
 	apt-get --no-install-recommends -y install \
 	git build-essential maven libpq-dev postgresql-common openjdk-8-jdk wget \
@@ -20,51 +22,34 @@ RUN npm i -g yarn
 RUN cp /usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/tools.jar && \
 	useradd -ms /bin/bash -d /apollo apollo
 
-ENV WEBAPOLLO_VERSION develop
-#RUN ls -la /apollo/
+RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat && \
+ 		chmod +x /usr/local/bin/blat && \
+ 		curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit && \
+ 		chmod +x /usr/local/bin/faToTwoBit && \
+		wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins26/chado-1.31.sql.gz -O /chado.sql.gz && \
+		gunzip /chado.sql.gz
+
 #NOTE, we had problems with the build the archive-file coming in from github so using a clone instead
-#RUN curl -L https://github.com/GMOD/Apollo/archive/${WEBAPOLLO_VERSION}.tar.gz | tar xzf - --strip-components=1 -C /apollo
-# RUN git clone --depth 1 --single-branch --branch ${WEBAPOLLO_VERSION} https://github.com/gmod/apollo /apollo/apollo-clone
-# RUN mv /apollo/apollo-clone/* /apollo && rm -rf /apollo/apollo-clone
 ADD . /apollo
-RUN rm -rf .git .gitignore
-#COPY * /apollo
 
 # install grails
 COPY docker-files/build.sh /bin/build.sh
 ADD docker-files/docker-apollo-config.groovy /apollo/apollo-config.groovy
 
 RUN chown -R apollo:apollo /apollo
-RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat
-RUN chmod +x /usr/local/bin/blat
-RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit
-RUN chmod +x /usr/local/bin/faToTwoBit
-
 
 USER apollo
-RUN curl -s get.sdkman.io | bash
-RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5"
-RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1"
-
-
-RUN /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh"
+RUN curl -s get.sdkman.io | bash && \
+		/bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" && \
+ 		/bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" && \
+ 		/bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh"
 
 USER root
-ENV CATALINA_HOME=/usr/local/tomcat
 RUN rm -rf ${CATALINA_HOME}/webapps/* && \
 	cp /apollo/apollo*.war ${CATALINA_HOME}/apollo.war
 
-ENV CONTEXT_PATH ROOT
-
-# Download chado schema
-RUN wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins97/chado-1.31.sql.gz -O /chado.sql.gz && \
-	gunzip /chado.sql.gz
-
-
 ADD docker-files/createenv.sh /createenv.sh
 CMD "/createenv.sh"
 
 ADD docker-files/launch.sh /launch.sh
 CMD "/launch.sh"
-
-

grails-app/controllers/org/bbop/apollo/OrganismController.groovy

@@ -101,7 +101,7 @@ class OrganismController {
         assert directoryToRemove.deleteDir()
       }
 
-      render findAllOrganisms()
+      findAllOrganisms()
 
     }
     catch (Exception e) {
@@ -1204,6 +1204,7 @@ class OrganismController {
       def c = Sequence.createCriteria()
       sequenceList = c.list {
         eq('organism', organism)
+        order('name',"asc")
       }
       log.debug "Sequence list fetched at getSequencesForOrganism: ${sequenceList}"
     } else {
@@ -1271,6 +1272,7 @@ class OrganismController {
   def updateOrganismInfo() {
     try {
       JSONObject organismJson = permissionService.handleInput(request, params)
+      println "updating organism info ${organismJson}"
       permissionService.checkPermissions(organismJson, PermissionEnum.ADMINISTRATE)
       Organism organism = Organism.findById(organismJson.id)
       Boolean madeObsolete
@@ -1289,9 +1291,15 @@ class OrganismController {
         madeObsolete = !organism.obsolete && (organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false)
         organism.obsolete = organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false
         organism.nonDefaultTranslationTable = organismJson.nonDefaultTranslationTable ?: organism.nonDefaultTranslationTable
+        println "Genome FASTa ${organism.genomeFasta}"
         if (organism.genomeFasta) {
           // update location of genome fasta
+          println "is a genome fasta"
           sequenceService.updateGenomeFasta(organism)
+          println "done is a genome fasta"
+        }
+        else{
+          println "is NOT a genome fasta"
         }
 
         CommonsMultipartFile organismDataFile = null
@@ -1325,9 +1333,12 @@ class OrganismController {
           }
           organism.save(flush: true, insert: false, failOnError: true)
 
-          if (oldOrganismDirectory!=organism.directory && !noReloadSequencesIfOrganismChanges) {
+          println "should be laoding a data file here ${organismDataFile} , ${oldOrganismDirectory}, ${organism.directory} , ${noReloadSequencesIfOrganismChanges}"
+          if ((organismDataFile || oldOrganismDirectory!=organism.directory) && !noReloadSequencesIfOrganismChanges) {
             // we need to reload
+            println "reloading refSeq"
             sequenceService.loadRefSeqs(organism)
+            println "DONE refSeq"
           }
         } else {
           throw new Exception("Bad organism directory: " + organism.directory)
@@ -1337,7 +1348,7 @@ class OrganismController {
       } else {
         throw new Exception('organism not found')
       }
-      render findAllOrganisms() as JSON
+      findAllOrganisms()
     }
     catch (e) {
       def error = [error: 'problem saving organism: ' + e]
@@ -1414,6 +1425,7 @@ class OrganismController {
       JSONObject requestObject = permissionService.handleInput(request, params)
       Boolean showPublicOnly = requestObject.showPublicOnly ? Boolean.valueOf(requestObject.showPublicOnly) : false
       Boolean showObsolete = requestObject.showObsolete ? Boolean.valueOf(requestObject.showObsolete) : false
+      println "find all organisms request object ${requestObject}"
       List<Organism> organismList = []
       if (requestObject.organism) {
         log.debug "finding info for specific organism"

grails-app/domain/org/bbop/apollo/Sequence.groovy

@@ -20,7 +20,6 @@ class Sequence {
     ]
 
     static mapping = {
-        cache usage: 'read-only'
         end column: 'sequence_end'
         start column: 'sequence_start'
         featureLocations cascade: 'all-delete-orphan'

grails-app/services/org/bbop/apollo/SequenceService.groovy

@@ -197,7 +197,7 @@ class SequenceService {
                 }
                 currentOffset -= alterationLength;
             }
-            // Substitions
+            // Substitutions
             else if (sequenceAlteration.instanceOf == SubstitutionArtifact.canonicalName) {
                 int start = strand == Strand.NEGATIVE ? localCoordinate - (alterationLength - 1) : localCoordinate;
                 residues.replace(start + currentOffset,
@@ -306,10 +306,14 @@ class SequenceService {
 
     def loadRefSeqs(Organism organism) {
         JSONObject referenceTrackObject = getReferenceTrackObject(organism)
+        println "loading ref sequences, ${referenceTrackObject}"
         if ((referenceTrackObject.storeClass == "JBrowse/Store/SeqFeature/IndexedFasta") || (referenceTrackObject.storeClass == "JBrowse/Store/Sequence/IndexedFasta")) {
+          println "A loading genome FASTA, ${referenceTrackObject}"
             loadGenomeFasta(organism, referenceTrackObject)
         } else {
+          println "B loading refSeq , ${referenceTrackObject}"
             loadRefSeqsJson(organism)
+          println "C LOADED refSeq , ${organism}"
         }
     }
 
@@ -325,9 +329,16 @@ class SequenceService {
             }
 
           def sequences = Sequence.findAllByOrganism(organism)
-          def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s join s.organism o where o = :organism",[organism:organism])
-          Preference.deleteAll(preferences)
-          Sequence.deleteAll(sequences)
+          def seqsMap = [:]
+          sequences.each { sequence ->
+              seqsMap[sequence.name] = sequence.length
+          }
+
+//          def knownSequences = Sequence.findAllByOrganism(organism)
+//          def seqsMap = [:]
+//          knownSequences.each { sequence ->
+//              seqsMap[sequence.name] = sequence.length
+//          }
 
           // this will fail if folks have actively been working on this and preferences are set
           // otherwise if we remove all of the sequences annotations will need to be removed as well
@@ -341,14 +352,41 @@ class SequenceService {
                     //workaround for jbrowse refSeqs that have no length element
                     length = refSeq.end - refSeq.start
                 }
-                Sequence sequence = new Sequence(
-                        organism: organism
-                        , length: length
-                        , seqChunkSize: refSeq.seqChunkSize
-                        , start: refSeq.start
-                        , end: refSeq.end
-                        , name: refSeq.name
-                ).save(failOnError: true)
+                if (!seqsMap.containsKey(refSeq.name)) {
+                    Sequence sequence = new Sequence(
+                            organism: organism
+                            , length: length
+                            , seqChunkSize: refSeq.seqChunkSize
+                            , start: refSeq.start
+                            , end: refSeq.end
+                            , name: refSeq.name
+                    ).save(failOnError: true)
+                    log.debug "added sequence ${sequence}"
+                }
+                else if (seqsMap[refSeq.name] != length) {
+                  Sequence sequence = Sequence.findByNameAndOrganism(refSeq.name,organism)
+                  sequence.length = length
+                  sequence.seqChunkSize = refSeq.seqChunkSize
+                  sequence.start = refSeq.start
+                  sequence.end = refSeq.end
+//                  sequence.name = refSeq.name
+                  sequence.save(failOnError: true,insert:false)
+//                    def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s where s = :sequence",[sequence:seqsMap[refSeq.name]])
+//                    Preference.deleteAll(preferences)
+//                    Sequence.delete(seqsMap[refSeq.name])
+//                    Sequence sequence = new Sequence(
+//                            organism: organism
+//                            , length: length
+//                            , seqChunkSize: refSeq.seqChunkSize
+//                            , start: refSeq.start
+//                            , end: refSeq.end
+//                            , name: refSeq.name
+//                    ).save(failOnError: true)
+                    log.debug "uddated sequence ${sequence}"
+                }
+                else {
+                    log.debug "skipped existing unchanged sequence ${refSeq.name}"
+                }
             }
 
             organism.valid = true
@@ -372,18 +410,47 @@ class SequenceService {
                 FastaSequenceIndex index = new FastaSequenceIndex(genomeFastaIndexFile)
                 log.info "an indexed fasta ${index}"
                 log.info "an indexed fasta size ${index.size()}"
+                def knownSequences = Sequence.findAllByOrganism(organism)
+                def seqsMap = [:]
+                knownSequences.each { sequence ->
+                    seqsMap[sequence.name] = sequence.length
+                }
                 // reading the index
                 def iterator = index.iterator()
                 while (iterator.hasNext()) {
                     def entry = iterator.next()
-                    Sequence sequence = new Sequence(
-                            organism: organism,
-                            length: entry.size,
-                            start: 0,
-                            end: entry.size,
-                            name: entry.contig
-                    ).save(failOnError: true)
-                    log.debug "added sequence ${sequence}"
+                  if (!seqsMap.containsKey(entry.contig)) {
+//                    if (!seqsMap.containsKey(entry.contig) || seqsMap[entry.contig] != entry.size) {
+                        Sequence sequence = new Sequence(
+                                organism: organism,
+                                length: entry.size,
+                                start: 0,
+                                end: entry.size,
+                                name: entry.contig
+                        ).save(failOnError: true,insert: true)
+                        log.debug "added sequence ${sequence}"
+                    }
+                    else if (seqsMap[entry.contig] != entry.size) {
+//                      def preferences = Preference.executeQuery("select p from UserOrganismPreference  p join p.sequence s where s = :sequence",[sequence:sequence])
+//                      Preference.deleteAll(preferences)
+//                      Sequence.delete(seqsMap[entry.contig])
+//                      Sequence sequence = new Sequence(
+//                        organism: organism,
+//                        length: entry.size,
+//                        start: 0,
+//                        end: entry.size,
+//                        name: entry.contig
+//                      ).save(failOnError: true)
+                      Sequence sequence = Sequence.findByNameAndOrganism(entry.contig,organism)
+                      sequence.length = entry.size as Integer
+                      sequence.start = 0
+                      sequence.end = entry.size as Integer
+                      sequence.save(failOnError: true,insert:false)
+                    log.debug "replaced sequence ${sequence}"
+                    }
+                    else {
+                        log.debug "skipped existing unchanged sequence ${entry.contig}"
+                    }
                 }
 
                 organism.valid = true

src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java

@@ -52,7 +52,7 @@ public Collection<BlastAlignment> search(String uniqueToken, String query, Strin
               p = Files.createTempDirectory(new File(tmpDir).toPath(),"blat_tmp");
           }
           dir = p.toFile();
-          
+
           return runSearch(dir, query, databaseId);
         }
         catch (IOException e) {