diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..5fedadb675 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +.gitignore +.github diff --git a/.travis.yml b/.travis.yml index 9443f42cb2..ffabef6174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,3 +80,7 @@ script: python setup.py nosetests killall java || true fi + +jobs: + allow_failures: + - env: DB=h2 TEST_SUITE=python-apollo diff --git a/Dockerfile b/Dockerfile index 91068e516a..be686ce4bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ -# WebApollo -# VERSION 2.1.X +# Apollo2 FROM tomcat:9-jdk8 MAINTAINER Nathan Dunn ENV DEBIAN_FRONTEND noninteractive +ENV CATALINA_HOME=/usr/local/tomcat +ENV CONTEXT_PATH ROOT + RUN apt-get -qq update --fix-missing && \ apt-get --no-install-recommends -y install \ git build-essential maven libpq-dev postgresql-common openjdk-8-jdk wget \ @@ -20,51 +22,34 @@ RUN npm i -g yarn RUN cp /usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar /usr/lib/jvm/java-8-openjdk-amd64/jre/lib/ext/tools.jar && \ useradd -ms /bin/bash -d /apollo apollo -ENV WEBAPOLLO_VERSION develop -#RUN ls -la /apollo/ +RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat && \ + chmod +x /usr/local/bin/blat && \ + curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit && \ + chmod +x /usr/local/bin/faToTwoBit && \ + wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins26/chado-1.31.sql.gz -O /chado.sql.gz && \ + gunzip /chado.sql.gz + #NOTE, we had problems with the build the archive-file coming in from github so using a clone instead -#RUN curl -L https://github.com/GMOD/Apollo/archive/${WEBAPOLLO_VERSION}.tar.gz | tar xzf - --strip-components=1 -C /apollo -# RUN git clone --depth 1 --single-branch --branch ${WEBAPOLLO_VERSION} https://github.com/gmod/apollo /apollo/apollo-clone -# RUN mv /apollo/apollo-clone/* /apollo && rm -rf /apollo/apollo-clone ADD . /apollo -RUN rm -rf .git .gitignore -#COPY * /apollo # install grails COPY docker-files/build.sh /bin/build.sh ADD docker-files/docker-apollo-config.groovy /apollo/apollo-config.groovy RUN chown -R apollo:apollo /apollo -RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/blat/blat" -o /usr/local/bin/blat -RUN chmod +x /usr/local/bin/blat -RUN curl -s "http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/faToTwoBit" -o /usr/local/bin/faToTwoBit -RUN chmod +x /usr/local/bin/faToTwoBit - USER apollo -RUN curl -s get.sdkman.io | bash -RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" -RUN /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" - - -RUN /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh" +RUN curl -s get.sdkman.io | bash && \ + /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install grails 2.5.5" && \ + /bin/bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && yes | sdk install gradle 3.2.1" && \ + /bin/bash -c "source $HOME/.profile && source $HOME/.sdkman/bin/sdkman-init.sh && /bin/bash /bin/build.sh" USER root -ENV CATALINA_HOME=/usr/local/tomcat RUN rm -rf ${CATALINA_HOME}/webapps/* && \ cp /apollo/apollo*.war ${CATALINA_HOME}/apollo.war -ENV CONTEXT_PATH ROOT - -# Download chado schema -RUN wget --quiet https://github.com/erasche/chado-schema-builder/releases/download/1.31-jenkins97/chado-1.31.sql.gz -O /chado.sql.gz && \ - gunzip /chado.sql.gz - - ADD docker-files/createenv.sh /createenv.sh CMD "/createenv.sh" ADD docker-files/launch.sh /launch.sh CMD "/launch.sh" - - diff --git a/grails-app/controllers/org/bbop/apollo/OrganismController.groovy b/grails-app/controllers/org/bbop/apollo/OrganismController.groovy index d96ec58b1a..9759777bbf 100644 --- a/grails-app/controllers/org/bbop/apollo/OrganismController.groovy +++ b/grails-app/controllers/org/bbop/apollo/OrganismController.groovy @@ -101,7 +101,7 @@ class OrganismController { assert directoryToRemove.deleteDir() } - render findAllOrganisms() + findAllOrganisms() } catch (Exception e) { @@ -1204,6 +1204,7 @@ class OrganismController { def c = Sequence.createCriteria() sequenceList = c.list { eq('organism', organism) + order('name',"asc") } log.debug "Sequence list fetched at getSequencesForOrganism: ${sequenceList}" } else { @@ -1271,6 +1272,7 @@ class OrganismController { def updateOrganismInfo() { try { JSONObject organismJson = permissionService.handleInput(request, params) + println "updating organism info ${organismJson}" permissionService.checkPermissions(organismJson, PermissionEnum.ADMINISTRATE) Organism organism = Organism.findById(organismJson.id) Boolean madeObsolete @@ -1289,9 +1291,15 @@ class OrganismController { madeObsolete = !organism.obsolete && (organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false) organism.obsolete = organismJson.containsKey("obsolete") ? Boolean.valueOf(organismJson.obsolete as String) : false organism.nonDefaultTranslationTable = organismJson.nonDefaultTranslationTable ?: organism.nonDefaultTranslationTable + println "Genome FASTa ${organism.genomeFasta}" if (organism.genomeFasta) { // update location of genome fasta + println "is a genome fasta" sequenceService.updateGenomeFasta(organism) + println "done is a genome fasta" + } + else{ + println "is NOT a genome fasta" } CommonsMultipartFile organismDataFile = null @@ -1325,9 +1333,12 @@ class OrganismController { } organism.save(flush: true, insert: false, failOnError: true) - if (oldOrganismDirectory!=organism.directory && !noReloadSequencesIfOrganismChanges) { + println "should be laoding a data file here ${organismDataFile} , ${oldOrganismDirectory}, ${organism.directory} , ${noReloadSequencesIfOrganismChanges}" + if ((organismDataFile || oldOrganismDirectory!=organism.directory) && !noReloadSequencesIfOrganismChanges) { // we need to reload + println "reloading refSeq" sequenceService.loadRefSeqs(organism) + println "DONE refSeq" } } else { throw new Exception("Bad organism directory: " + organism.directory) @@ -1337,7 +1348,7 @@ class OrganismController { } else { throw new Exception('organism not found') } - render findAllOrganisms() as JSON + findAllOrganisms() } catch (e) { def error = [error: 'problem saving organism: ' + e] @@ -1414,6 +1425,7 @@ class OrganismController { JSONObject requestObject = permissionService.handleInput(request, params) Boolean showPublicOnly = requestObject.showPublicOnly ? Boolean.valueOf(requestObject.showPublicOnly) : false Boolean showObsolete = requestObject.showObsolete ? Boolean.valueOf(requestObject.showObsolete) : false + println "find all organisms request object ${requestObject}" List organismList = [] if (requestObject.organism) { log.debug "finding info for specific organism" diff --git a/grails-app/domain/org/bbop/apollo/Sequence.groovy b/grails-app/domain/org/bbop/apollo/Sequence.groovy index 65acc582ab..a9e8b3a905 100644 --- a/grails-app/domain/org/bbop/apollo/Sequence.groovy +++ b/grails-app/domain/org/bbop/apollo/Sequence.groovy @@ -20,7 +20,6 @@ class Sequence { ] static mapping = { - cache usage: 'read-only' end column: 'sequence_end' start column: 'sequence_start' featureLocations cascade: 'all-delete-orphan' diff --git a/grails-app/services/org/bbop/apollo/SequenceService.groovy b/grails-app/services/org/bbop/apollo/SequenceService.groovy index 24d420930a..af392eb0c6 100644 --- a/grails-app/services/org/bbop/apollo/SequenceService.groovy +++ b/grails-app/services/org/bbop/apollo/SequenceService.groovy @@ -197,7 +197,7 @@ class SequenceService { } currentOffset -= alterationLength; } - // Substitions + // Substitutions else if (sequenceAlteration.instanceOf == SubstitutionArtifact.canonicalName) { int start = strand == Strand.NEGATIVE ? localCoordinate - (alterationLength - 1) : localCoordinate; residues.replace(start + currentOffset, @@ -306,10 +306,14 @@ class SequenceService { def loadRefSeqs(Organism organism) { JSONObject referenceTrackObject = getReferenceTrackObject(organism) + println "loading ref sequences, ${referenceTrackObject}" if ((referenceTrackObject.storeClass == "JBrowse/Store/SeqFeature/IndexedFasta") || (referenceTrackObject.storeClass == "JBrowse/Store/Sequence/IndexedFasta")) { + println "A loading genome FASTA, ${referenceTrackObject}" loadGenomeFasta(organism, referenceTrackObject) } else { + println "B loading refSeq , ${referenceTrackObject}" loadRefSeqsJson(organism) + println "C LOADED refSeq , ${organism}" } } @@ -325,9 +329,16 @@ class SequenceService { } def sequences = Sequence.findAllByOrganism(organism) - def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s join s.organism o where o = :organism",[organism:organism]) - Preference.deleteAll(preferences) - Sequence.deleteAll(sequences) + def seqsMap = [:] + sequences.each { sequence -> + seqsMap[sequence.name] = sequence.length + } + +// def knownSequences = Sequence.findAllByOrganism(organism) +// def seqsMap = [:] +// knownSequences.each { sequence -> +// seqsMap[sequence.name] = sequence.length +// } // this will fail if folks have actively been working on this and preferences are set // otherwise if we remove all of the sequences annotations will need to be removed as well @@ -341,14 +352,41 @@ class SequenceService { //workaround for jbrowse refSeqs that have no length element length = refSeq.end - refSeq.start } - Sequence sequence = new Sequence( - organism: organism - , length: length - , seqChunkSize: refSeq.seqChunkSize - , start: refSeq.start - , end: refSeq.end - , name: refSeq.name - ).save(failOnError: true) + if (!seqsMap.containsKey(refSeq.name)) { + Sequence sequence = new Sequence( + organism: organism + , length: length + , seqChunkSize: refSeq.seqChunkSize + , start: refSeq.start + , end: refSeq.end + , name: refSeq.name + ).save(failOnError: true) + log.debug "added sequence ${sequence}" + } + else if (seqsMap[refSeq.name] != length) { + Sequence sequence = Sequence.findByNameAndOrganism(refSeq.name,organism) + sequence.length = length + sequence.seqChunkSize = refSeq.seqChunkSize + sequence.start = refSeq.start + sequence.end = refSeq.end +// sequence.name = refSeq.name + sequence.save(failOnError: true,insert:false) +// def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s where s = :sequence",[sequence:seqsMap[refSeq.name]]) +// Preference.deleteAll(preferences) +// Sequence.delete(seqsMap[refSeq.name]) +// Sequence sequence = new Sequence( +// organism: organism +// , length: length +// , seqChunkSize: refSeq.seqChunkSize +// , start: refSeq.start +// , end: refSeq.end +// , name: refSeq.name +// ).save(failOnError: true) + log.debug "uddated sequence ${sequence}" + } + else { + log.debug "skipped existing unchanged sequence ${refSeq.name}" + } } organism.valid = true @@ -372,18 +410,47 @@ class SequenceService { FastaSequenceIndex index = new FastaSequenceIndex(genomeFastaIndexFile) log.info "an indexed fasta ${index}" log.info "an indexed fasta size ${index.size()}" + def knownSequences = Sequence.findAllByOrganism(organism) + def seqsMap = [:] + knownSequences.each { sequence -> + seqsMap[sequence.name] = sequence.length + } // reading the index def iterator = index.iterator() while (iterator.hasNext()) { def entry = iterator.next() - Sequence sequence = new Sequence( - organism: organism, - length: entry.size, - start: 0, - end: entry.size, - name: entry.contig - ).save(failOnError: true) - log.debug "added sequence ${sequence}" + if (!seqsMap.containsKey(entry.contig)) { +// if (!seqsMap.containsKey(entry.contig) || seqsMap[entry.contig] != entry.size) { + Sequence sequence = new Sequence( + organism: organism, + length: entry.size, + start: 0, + end: entry.size, + name: entry.contig + ).save(failOnError: true,insert: true) + log.debug "added sequence ${sequence}" + } + else if (seqsMap[entry.contig] != entry.size) { +// def preferences = Preference.executeQuery("select p from UserOrganismPreference p join p.sequence s where s = :sequence",[sequence:sequence]) +// Preference.deleteAll(preferences) +// Sequence.delete(seqsMap[entry.contig]) +// Sequence sequence = new Sequence( +// organism: organism, +// length: entry.size, +// start: 0, +// end: entry.size, +// name: entry.contig +// ).save(failOnError: true) + Sequence sequence = Sequence.findByNameAndOrganism(entry.contig,organism) + sequence.length = entry.size as Integer + sequence.start = 0 + sequence.end = entry.size as Integer + sequence.save(failOnError: true,insert:false) + log.debug "replaced sequence ${sequence}" + } + else { + log.debug "skipped existing unchanged sequence ${entry.contig}" + } } organism.valid = true diff --git a/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java b/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java index a13e70da81..9fcafff3cf 100644 --- a/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java +++ b/src/groovy/org/bbop/apollo/sequence/search/blat/BlatCommandLine.java @@ -52,7 +52,7 @@ public Collection search(String uniqueToken, String query, Strin p = Files.createTempDirectory(new File(tmpDir).toPath(),"blat_tmp"); } dir = p.toFile(); - + return runSearch(dir, query, databaseId); } catch (IOException e) {