diff --git a/.travis.yml b/.travis.yml index 89efc1041..abb876897 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,55 @@ language: groovy -sudo: false jdk: - - oraclejdk8 + - oraclejdk8 +php: + - 5.6 +env: + - TERM=dumb; DIR=transmart-core-db-tests; PREPARE=assemble; TEST=check + - TERM=dumb; DIR=transmart-data; PREINSTALL="createVars setupPostgres"; INSTALL=downloadTransmartETL; TEST=dataloadingTest + +addons: + postgres: 9.4 before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock + - rm -rf $HOME/.gradle/caches/*/plugin-resolution/ cache: directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - - $HOME/.m2/ + - $HOME/.gradle/caches + - $HOME/.gradle/wrapper + - $HOME/.m2 + - $HOME/pg + +before_install: + - gradle --version + - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 3375DA21 + - echo deb http://apt.thehyve.net/internal/ trusty main | sudo tee /etc/apt/sources.list.d/hyve_internal.list + - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E5267A6C + - echo deb http://ppa.launchpad.net/ondrej/php5/ubuntu precise main | sudo tee /etc/apt/sources.list.d/ondrej_php5.list + - sudo apt-get -qq update + - apt-cache search php + - sudo apt-get install -y tk-dev php5-cli php5-json + - php --version + - echo $DIR; cd $DIR + - PWD=`pwd` + - echo $PWD + - ([ -z "$PREINSTALL" ] && echo "Skipping preinstall.") || (echo $PREINSTALL; sudo -i -- sh -c "cd $PWD; gradle $PREINSTALL") + - cd - + +install: + - echo $DIR; cd $DIR + - ([ -z "$INSTALL" ] && echo "Skipping install.") || (echo $INSTALL; gradle $INSTALL) + - cd - before_script: - - cd transmart-core-db-tests - - ./gradlew --console=plain assemble + - echo $DIR; cd $DIR + - ([ -z "$PREPARE" ] && echo "Skipping prepare.") || (echo $PREPARE; gradle $PREPARE) - cd - script: - - cd transmart-core-db-tests - - ./gradlew --console=plain --stacktrace check + - echo $DIR; cd $DIR + - ([ -z "$TEST" ] && echo "Skipping test.") || (echo $TEST; gradle --stacktrace $TEST) - cd - notifications: @@ -27,5 +57,5 @@ notifications: rooms: secure: "rNPPKSL1o8R71YcD7T+jDFTFfSFoXO/YKCubqwP5TXFL+OtqEwZOrkpKSpyN+Ap6Uxk85TBgat8Bur/WkTWXcq6HruyhfKkKea96ZVVSWjZWhKHqqTLvMeABb1uh6JxpXAc/5Cm3K5/yTHBFFiFSyOemuEl88bqeqJjDSzIxvOLrdLhnYWVlo3SJcUwAab0IKab8MpbZUEa6eFB5+MmCQfwjDZBr9Qn7UGfQWQdEp801sHGzU2i58uwF7+RVYhUyZtT9428FmPhxAS2EgW663JpzNQYpVCws02UW8aCgu2SCWWtUKwk3dy/6hSc5UoxWIIKaIpPbre+SA5JYeB7RwIad7N6/eDHdb/ByN1lOP6/pAEzPSnajxJrU/AnohhIwgcPaZ3aHbrqq8RrGPy1sNwFTKvFzBkVu5aWKR7JMgKuBq0LXv4iq1qBJw1s+xK4CHNoOAESZG2a0wo/cmzK+C2OzKdTnjN25RoHwXBOS7AzJtdtqpU8PaESZ0X1BxT19nD3RALiCfS/054SU3mfswhVAdo6rVhHDgGNxqRflgZIVmDsYjBRZ4TR4jDnUhc+s9ILo1eI/imq7tQZ6rFeQrydCu4p2U4LrFAWu7E0QwTbsfnSs5KN7Fah1xhS+6xttOuqbAmPQXEeLKSkk6B9Q7G/R0VHAKdaNkLweA0ZzATM=" on_success: change - on_failure: always + on_failure: change diff --git a/Rmodules/build.gradle b/Rmodules/build.gradle index 90e98d6da..06becb1e0 100644 --- a/Rmodules/build.gradle +++ b/Rmodules/build.gradle @@ -75,7 +75,8 @@ dependencies { testCompile 'org.gmock:gmock:0.8.3' testCompile 'org.hamcrest:hamcrest-library:1.3' testCompile 'org.hamcrest:hamcrest-core:1.3' - compile 'quartz:quartz:1.5.1-atlassian-2' + + compile 'org.grails.plugins:quartz:2.0.1' // https://mvnrepository.com/artifact/org.apache.commons/commons-io compile group: 'org.apache.commons', name: 'commons-io', version: '1.3.2' // https://mvnrepository.com/artifact/org.xhtmlrenderer/flying-saucer-pdf diff --git a/Rmodules/grails-app/conf/org/transmartproject/rmodules/AuditLogFilters.groovy b/Rmodules/grails-app/conf/org/transmartproject/rmodules/AuditLogFilters.groovy deleted file mode 100644 index 592e7e934..000000000 --- a/Rmodules/grails-app/conf/org/transmartproject/rmodules/AuditLogFilters.groovy +++ /dev/null @@ -1,23 +0,0 @@ -package org.transmartproject.rmodules - -import org.transmartproject.core.users.User - -class AuditLogFilters { - - def accessLogService - User currentUserBean - - def filters = { - download(controller: 'analysisFiles', action:'download') { - after = { model -> - if (params.path.toLowerCase().endsWith('.zip')) { - def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr - accessLogService.report(currentUserBean, 'Raw R Data Export', - eventMessage: "User (IP: ${ip}) downloaded ${params.path} for ${params.analysisName}", - requestURL: "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}") - } - } - } - } - -} diff --git a/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/DataAssociationController.groovy b/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/DataAssociationController.groovy index 888d45088..369b49bcd 100644 --- a/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/DataAssociationController.groovy +++ b/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/DataAssociationController.groovy @@ -40,8 +40,8 @@ class DataAssociationController { // list of required css def styles = [ - servletContext.contextPath+pluginContextPath+'/css/rmodules.css', - servletContext.contextPath+pluginContextPath+'/css/jquery.qtip.min.css'] + servletContext.contextPath+pluginContextPath+'/assets/rmodules.css', + servletContext.contextPath+pluginContextPath+'/assets/jquery.qtip.min.css'] //TODO: requires images: servletContext.contextPath+pluginContextPath+'/css/jquery-ui-1.10.3.custom.css'] JSONObject result = new JSONObject() diff --git a/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/RModulesController.groovy b/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/RModulesController.groovy index 5b3fc616f..774c67df6 100644 --- a/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/RModulesController.groovy +++ b/Rmodules/grails-app/controllers/com/recomdata/transmart/data/association/RModulesController.groovy @@ -19,15 +19,17 @@ package com.recomdata.transmart.data.association import com.google.common.collect.Maps import grails.converters.JSON import grails.core.GrailsApplication -import grails.util.Holders import jobs.* import jobs.misc.AnalysisConstraints import jobs.misc.AnalysisQuartzJobAdapter import org.grails.web.converters.exceptions.ConverterException import org.grails.web.json.JSONElement +import org.quartz.JobBuilder import org.quartz.JobDataMap import org.quartz.JobDetail import org.quartz.SimpleTrigger +import org.quartz.TriggerBuilder +import org.quartz.core.QuartzScheduler import org.transmartproject.core.exceptions.InvalidArgumentsException import static jobs.misc.AnalysisQuartzJobAdapter.* @@ -55,6 +57,7 @@ class RModulesController { def RModulesService GrailsApplication grailsApplication def jobResultsService + QuartzScheduler quartzScheduler /** * Method called that will cancel a running job @@ -169,9 +172,14 @@ class RModulesController { params.put(PARAM_USER_PARAMETERS, userParams) params.put(PARAM_USER_IN_CONTEXT, currentUserBean.targetSource.target) - JobDetail jobDetail = new JobDetail(params.jobName, params.jobType, AnalysisQuartzJobAdapter) - jobDetail.jobDataMap = new JobDataMap(params) - SimpleTrigger trigger = new SimpleTrigger("triggerNow ${Calendar.instance.time.time}", 'RModules') + JobDetail jobDetail = JobBuilder.newJob(AnalysisQuartzJobAdapter) + .withIdentity(params.jobName, params.jobType) + .setJobData(params) + .build() + SimpleTrigger trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity('RModules') + .build() quartzScheduler.scheduleJob(jobDetail, trigger) } @@ -226,7 +234,4 @@ class RModulesController { constraints } - private static def getQuartzScheduler() { - Holders.grailsApplication.mainContext.quartzScheduler - } } diff --git a/Rmodules/grails-app/controllers/rmodules/AnalysisFilesInterceptor.groovy b/Rmodules/grails-app/controllers/rmodules/AnalysisFilesInterceptor.groovy new file mode 100644 index 000000000..297a97a2a --- /dev/null +++ b/Rmodules/grails-app/controllers/rmodules/AnalysisFilesInterceptor.groovy @@ -0,0 +1,29 @@ +package rmodules + +import org.transmartproject.core.users.User + +class AnalysisFilesInterceptor { + + def accessLogService + User currentUserBean + + AnalysisFilesInterceptor() { + match(controller: 'analysisFiles', action:'download') + } + + boolean before() { true } + + boolean after() { + if (params.path.toLowerCase().endsWith('.zip')) { + def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr + accessLogService.report(currentUserBean, 'Raw R Data Export', + eventMessage: "User (IP: ${ip}) downloaded ${params.path} for ${params.analysisName}", + requestURL: "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}") + } + true + } + + void afterView() { + // no-op + } +} diff --git a/Rmodules/grails-app/controllers/rdc/rmodules/UrlMappings.groovy b/Rmodules/grails-app/controllers/rmodules/UrlMappings.groovy similarity index 93% rename from Rmodules/grails-app/controllers/rdc/rmodules/UrlMappings.groovy rename to Rmodules/grails-app/controllers/rmodules/UrlMappings.groovy index a692f60c6..51b493f5b 100644 --- a/Rmodules/grails-app/controllers/rdc/rmodules/UrlMappings.groovy +++ b/Rmodules/grails-app/controllers/rmodules/UrlMappings.groovy @@ -1,4 +1,4 @@ -package rdc.rmodules +package rmodules class UrlMappings { diff --git a/Rmodules/grails-app/services/com/recomdata/transmart/data/association/RModulesService.groovy b/Rmodules/grails-app/services/com/recomdata/transmart/data/association/RModulesService.groovy index 2ffe818fc..2f49fbb47 100644 --- a/Rmodules/grails-app/services/com/recomdata/transmart/data/association/RModulesService.groovy +++ b/Rmodules/grails-app/services/com/recomdata/transmart/data/association/RModulesService.groovy @@ -17,14 +17,16 @@ package com.recomdata.transmart.data.association import com.recomdata.transmart.data.association.asynchronous.RModulesJobService +import grails.transaction.Transactional import org.grails.web.json.JSONObject +import org.quartz.JobBuilder import org.quartz.JobDataMap -import org.quartz.JobDetail -import org.quartz.SimpleTrigger +import org.quartz.TriggerBuilder +import org.quartz.core.QuartzScheduler +@Transactional class RModulesService { - static transactional = true static scope = 'request' static STATUS_LIST = [ @@ -39,8 +41,7 @@ class RModulesService { /** * quartzScheduler is available from the Quartz grails-plugin */ - def quartzScheduler - + QuartzScheduler quartzScheduler def jobResultsService def asyncJobService def grailsApplication @@ -211,16 +212,21 @@ class RModulesService { if (jobResultsService[params.jobName]["Status"] == "Cancelled") {return} //com.recomdata.transmart.plugin.PluginJobExecutionService should be implemented by all Plugins - def jobDetail = new JobDetail(params.jobName, params.jobType, RModulesJobService.class) + def jobDetail = JobBuilder.newJob(RModulesJobService.class) + .withIdentity(params.jobName, params.jobType) + .build() jobDetail.setJobDataMap(jobDataMap) if (asyncJobService.updateStatus(params.jobName, jobStatusList[2])) { return } - def trigger = new SimpleTrigger("triggerNow"+Calendar.instance.time.time, 'RModules') + def trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity('RModules') + .build() quartzScheduler.scheduleJob(jobDetail, trigger) } - + // method for non-R jobs def prepareDataForExport(userName, params) { loadJobDataMap(userName, params); diff --git a/Rmodules/grails-app/views/plugin/AcghFrequencyPlot.gsp b/Rmodules/grails-app/views/plugin/AcghFrequencyPlot.gsp index 18f0c4421..2b76b9243 100644 --- a/Rmodules/grails-app/views/plugin/AcghFrequencyPlot.gsp +++ b/Rmodules/grails-app/views/plugin/AcghFrequencyPlot.gsp @@ -1,6 +1,6 @@ %{--include js lib for heatmap dynamically--}% - - +%{----}% +%{----}% %{--Input Container--}%
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..2c6137b87 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..6ea49ab86 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Sep 27 10:49:32 CEST 2016 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.12-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 000000000..9d82f7891 --- /dev/null +++ b/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/transmart-batch/gradle/wrapper/gradle-wrapper.jar b/transmart-batch/gradle/wrapper/gradle-wrapper.jar index 3d0dee6e8..2c6137b87 100644 Binary files a/transmart-batch/gradle/wrapper/gradle-wrapper.jar and b/transmart-batch/gradle/wrapper/gradle-wrapper.jar differ diff --git a/transmart-batch/gradle/wrapper/gradle-wrapper.properties b/transmart-batch/gradle/wrapper/gradle-wrapper.properties index 3f5497227..e75156f5c 100644 --- a/transmart-batch/gradle/wrapper/gradle-wrapper.properties +++ b/transmart-batch/gradle/wrapper/gradle-wrapper.properties @@ -1,4 +1,4 @@ -#Fri Jul 10 11:57:20 CEST 2015 +#Tue Sep 27 10:49:32 CEST 2016 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME diff --git a/transmart-batch/gradlew b/transmart-batch/gradlew index 91a7e269e..9d82f7891 100755 --- a/transmart-batch/gradlew +++ b/transmart-batch/gradlew @@ -42,11 +42,6 @@ case "`uname`" in ;; esac -# For Cygwin, ensure paths are in UNIX format before anything is touched. -if $cygwin ; then - [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` -fi - # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" @@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do fi done SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >&- +cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" -cd "$SAVED" >&- +cd "$SAVED" >/dev/null CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -114,6 +109,7 @@ fi if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` diff --git a/transmart-batch/gradlew.bat b/transmart-batch/gradlew.bat index aec99730b..72d362daf 100755 --- a/transmart-batch/gradlew.bat +++ b/transmart-batch/gradlew.bat @@ -46,7 +46,7 @@ echo location of your Java installation. goto fail :init -@rem Get command-line arguments, handling Windowz variants +@rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args if "%@eval[2+2]" == "4" goto 4NT_args diff --git a/transmart-core-db/grails-app/services/org/transmartproject/db/querytool/QueriesResourceService.groovy b/transmart-core-db/grails-app/services/org/transmartproject/db/querytool/QueriesResourceService.groovy index 91ca95507..b32bfcb39 100644 --- a/transmart-core-db/grails-app/services/org/transmartproject/db/querytool/QueriesResourceService.groovy +++ b/transmart-core-db/grails-app/services/org/transmartproject/db/querytool/QueriesResourceService.groovy @@ -19,6 +19,7 @@ package org.transmartproject.db.querytool +import grails.transaction.Transactional import org.hibernate.jdbc.Work import org.transmartproject.core.exceptions.InvalidRequestException import org.transmartproject.core.exceptions.NoSuchResourceException @@ -30,6 +31,7 @@ import org.transmartproject.db.user.User import java.sql.Connection +@Transactional class QueriesResourceService implements QueriesResource { def grailsApplication diff --git a/transmart-data/build.gradle b/transmart-data/build.gradle index 4ca24c72f..b895c7ddd 100644 --- a/transmart-data/build.gradle +++ b/transmart-data/build.gradle @@ -15,19 +15,69 @@ project.ext { environment = [PATH: '/bin/:/usr/bin/:/usr/local/bin/'] } -task createVars(type:Exec) { +def make(String target) { + println "make: ${target}" + exec { + commandLine 'bash', '-c', 'source ./vars && make ' + target + } +} + +def sudo_make(String target) { + println "sudo make: ${target}" + exec { + commandLine 'bash', '-c', 'source ./vars && sudo make ' + target + } +} + +task checkDependencies << { + def out = new ByteArrayOutputStream() + exec { + commandLine 'php', '--version' + standardOutput out + } + println out.toString().readLines()[0] + def php_installed = false + out.toString().find(/PHP (\d+).(\d+)/) { match, major, minor -> + if (major > 5 || major == 5 && minor >= 4) { + php_installed = true + } + } + if (!php_installed) { + throw new StopActionException('PHP version >= 5.4 required.') + } + out.reset() + exec { + commandLine 'svn', '--version' + standardOutput out + } + println out.toString().readLines()[0] +} + +task createVars(type: Exec) { commandLine 'make', '-C', 'env', '../vars' } createVars.onlyIf { !file('vars').exists() } +createVars.dependsOn 'checkDependencies' -def make(String target) { +def etl_target = 'env/tranSMART-ETL' + +task downloadTransmartETL << { + def repo = 'https://github.com/transmart/tranSMART-ETL/trunk' exec { - commandLine 'bash', '-c', 'source ./vars && make ' + target + commandLine 'svn', 'checkout', '--depth', 'immediates', repo, etl_target + } + exec { + commandLine 'svn', 'up', '--set-depth=infinity', "${etl_target}/Postgres" + } + exec { + commandLine 'svn', 'up', '--set-depth=infinity', "${etl_target}/Kettle-GPL" } - println target } +downloadTransmartETL.onlyIf { !file(etl_target).exists() } +downloadTransmartETL.dependsOn 'checkDependencies' task setupPostgres << { + sudo_make('-C env /var/lib/postgresql/tablespaces') make('postgres') } @@ -47,4 +97,11 @@ task stopSolr << { make('-C solr stop_solr') } +task dataloadingTest << { + make('-C env data-integration') + make('-C samples/postgres load_clinical_GSE8581') + make('-C samples/postgres showdblog') +} +dataloadingTest.dependsOn 'checkDependencies' + defaultTasks 'createVars' diff --git a/transmart-data/samples/studies/.gitignore b/transmart-data/samples/studies/.gitignore new file mode 100644 index 000000000..ebbbf6f67 --- /dev/null +++ b/transmart-data/samples/studies/.gitignore @@ -0,0 +1,2 @@ +/datasets +/*/ diff --git a/transmart-data/samples/studies/Makefile b/transmart-data/samples/studies/Makefile new file mode 100644 index 000000000..26a7f86f9 --- /dev/null +++ b/transmart-data/samples/studies/Makefile @@ -0,0 +1,81 @@ +include ../../lib/makefile.inc + +FEED_LISTS:=public-feeds $(wildcard ../../private-feeds) +datasets: $(FEED_LISTS) + groovy -cp '$(LIB_DIR)' build_datasets_index.groovy \ + $(foreach f,$(FEED_LISTS),-f $f) -o $@ + +update_datasets: + @rm -f datasets + @$(MAKE) --quiet datasets + +# depends on %/%_clinical.tar.xz, but make doesn't support repeated % +%/clinical.params %/clinical: + $(MAKE) $*/$*_clinical.tar.xz + tar -C $* -xJf $*/$*_clinical.tar.xz + +%/annotation.params %/annotation: + $(MAKE) $*/$*_annotation.tar.xz + tar -C $* -xJf $*/$*_annotation.tar.xz + +%/ref_annotation.params: + $(MAKE) $*/$*_ref_annotation.tar.xz + tar -C $* -xJf $*/$*_ref_annotation.tar.xz + +%/expression.params %/expression: + $(MAKE) $*/$*_expression.tar.xz + tar -C $* -xJf $*/$*_expression.tar.xz + +%/acgh.params %/acgh: + $(MAKE) $*/$*_acgh.tar.xz + tar -C $* -xJf $*/$*_acgh.tar.xz + +%/mirna.params: + $(MAKE) $*/$*_mirna.tar.xz + tar -C $* -xJf $*/$*_mirna.tar.xz + +%/mirnaqpcr.params %/mirnaqpcr: + $(MAKE) $*/$*_mirnaqpcr.tar.xz + tar -C $* -xJf $*/$*_mirnaqpcr.tar.xz + +%/mirnaseq.params %/mirnaseq: + $(MAKE) $*/$*_mirnaseq.tar.xz + tar -C $* -xJf $*/$*_mirnaseq.tar.xz + +%/proteomics.params: + $(MAKE) $*/$*_proteomics.tar.xz + tar -C $* -xJf $*/$*_proteomics.tar.xz + +%/msproteomics.params %/msproteomics: + $(MAKE) $*/$*_msproteomics.tar.xz + tar -C $* -xJf $*/$*_msproteomics.tar.xz + +%/metabolomics.params %/metabolomics: + $(MAKE) $*/$*_metabolomics.tar.xz + tar -C $* -xJf $*/$*_metabolomics.tar.xz + +%/rbm.params %/rbm: + $(MAKE) $*/$*_rbm.tar.xz + tar -C $* -xJf $*/$*_rbm.tar.xz + +%/rnaseq.params %.rnaseq: + $(MAKE) $*/$*_rnaseq.tar.xz + tar -C $* -xJf $*/$*_rnaseq.tar.xz + +%/vcf.params: + $(MAKE) $*/$*_vcf.tar.xz + tar -C $* -xJf $*/$*_vcf.tar.xz + +TARBALLS := $(shell test ! -f datasets || php enumerate.php tarballs) +$(TARBALLS): + @test -f datasets || $(MAKE) datasets + mkdir -p "$$(php -r 'preg_match("@^[^/]+@", $$argv[1], $$m); echo $$m[0];' '$@')" + ./download_tarball.sh '$@' $$(php get_file_location.php '$@') + +clean: + rm -f datasets + find . -path './*' -prune -type d -exec rm -r '{}' \; + +.PHONY: update_datasets clean + +.DELETE_ON_ERROR: diff --git a/transmart-data/samples/studies/build_datasets_index.groovy b/transmart-data/samples/studies/build_datasets_index.groovy new file mode 100644 index 000000000..124c8765c --- /dev/null +++ b/transmart-data/samples/studies/build_datasets_index.groovy @@ -0,0 +1,246 @@ +/* + * Copyright © 2013-2014 The Hyve B.V. + * + * This file is part of transmart-data. + * + * Transmart-data is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * transmart-data. If not, see . + */ + + +import groovy.transform.ToString +@Grab(group='org.codehaus.gpars', module='gpars', version='1.1.0') +import groovyx.gpars.GParsPool +@Grab(group='commons-net', module='commons-net', version='3.3') +import org.apache.commons.net.ftp.FTPClient +import groovy.transform.Canonical +import groovy.transform.InheritConstructors +import org.apache.commons.net.ftp.FTPReply +import groovy.util.logging.Slf4j +@Grab(group='ch.qos.logback', module='logback-classic', version='1.0.13') + +@Slf4j +class Script { + + static run(args) { + def cli = new CliBuilder() + cli.f 'Use the feeds specified in this file; can be specified multiple times', + args: 1, argName: 'file', longOpt: 'feed-list', required: true + cli.o 'Output file', args: 1, argName: 'file', longOpt: 'out', required: true + cli.h 'Show this help', longOpt: 'help' + cli.usage = '[-j ] -f [-f ...]' + + def options = cli.parse args + if ( !options ) { + log.error 'Invalid options' + System.exit 1 + } + final int NUMBER_OF_THREADS = 4 + + List dataSets + GParsPool.withPool ( NUMBER_OF_THREADS ) { + def feeds = new FeedFactory(options.fs).createFeeds() + dataSets = feeds.collect { feed -> + try { + def result = feed.fetchDataSets() + log.info "Got ${result.size()} data sets from $feed" + result + } catch (Exception e) { + log.error "Error fetching data sets for feed $feed: ${e.message}" + System.exit 1 + } + }.flatten() + } + + def groupedDataSets = dataSets.groupBy { + [it.study, it.type] + } + log.info "Found ${groupedDataSets.size()} unique data sets" + dataSets = groupedDataSets.collect { + List key, List list -> + list + } + + new File ( options.o ).withWriter { + writer -> + dataSets.each { + writer.write "${it[0].study} ${it[0].type} " + + "${it*.URL*.toExternalForm().join(" ")}\n" + } + } + } +} + +Script.run(args) + +interface Feed { + Set fetchDataSets() +} + +@Canonical(excludes = 'URL') +class DataSet { + String baseLocation + String relativePath + String study + String type + + URL getURL() { + new URL(baseLocation + relativePath) + } +} + +class FeedFactory { + List files + FeedFactory(List files) { + this.files = files.collect { new File(it) } + } + + final static Map FEED_TYPES = [ + 'http-index': HttpIndexFeed, + 'ftp-flat': FtpFlatFeed, + ] + + Set createFeeds() { + files.collect { + def result = [] + it.eachLine { line -> + List split = line.split(/\s+/, 2) as List + if (split.size() != 2) { + throw new InvalidDataException( + "Invalid line in feed list: $line") + } + if (!FEED_TYPES[split[0]]) { + throw new InvalidDataException( + "Unknown feed type: ${split[0]}") + } + result << FEED_TYPES[split[0]].newInstance(split[1]) + } + result + }.flatten() as Set + } +} + +@InheritConstructors +class InvalidDataException extends RuntimeException {} + +@ToString(includes = 'index') +class HttpIndexFeed implements Feed { + URL index + String baseLocation + + HttpIndexFeed(String data) { + index = new URL(data) + def urlString = index as String + baseLocation = urlString.substring(0, urlString.lastIndexOf('/') + 1) + } + + @Override + Set fetchDataSets() { + def ret = [] as Set + + def is = index.openStream() + try { + // assumes UTF-8 + def reader = new BufferedReader(new InputStreamReader(is, 'UTF-8')) + def line + while ((line = reader.readLine()) != null) { + if (line =~ /^\s*$/ || line =~ /^#/) { + continue + } + ret << parseLine(line) + } + } finally { + is.close() + } + + ret + } + + DataSet parseLine(String line) { + def split = line.split(/\s+/, 3) as List + if (split.size() != 3) { + throw new InvalidDataException("Invalid line from feed $index: $line") + } + new DataSet( + baseLocation: baseLocation, + relativePath: split[2], + study: split[0], + type: split[1] + ) + } +} + +@ToString(includes = 'url') +class FtpFlatFeed implements Feed { + URL url + FtpFlatFeed(String data) { + if (data[-1] != '/') { + data += '/' + } + url = new URL(data) + if (url.protocol != 'ftp') { + throw new InvalidDataException("Expected ftp url; got $data") + } + } + + @Override + Set fetchDataSets() { + def ftp = new FTPClient() + ftp.connect(url.host, url.port == -1 ? url.defaultPort : url.port) + checkReply ftp, 'connect' + + ftp.enterLocalPassiveMode() + + def userInfo = url.userInfo ?: 'anonymous:ftp@example.com' + + try { + if (userInfo) { + def userPass = userInfo.split(':', 2) as List + ftp.login(userPass[0], userPass[1] ?: '') + checkReply ftp, 'login' + } + ftp.cwd(url.path) + checkReply ftp, 'cwd' + + + def names = ftp.listNames() + if (names == null) { + throw new InvalidDataException("Error listing directory for $this") + } + + names.collect { name -> + def matcher = name =~ /^(.+)_(.+)\.tar\.xz$/ + if (!matcher.matches()) { + return + } + + new DataSet( + baseLocation: url.toExternalForm(), + relativePath: name, + study: matcher.group(1), + type: matcher.group(2), + ) + }.findAll() as Set + } finally { + ftp.disconnect() + } + } + + void checkReply(FTPClient client, String stage) { + def reply = client.replyCode + if (!FTPReply.isPositiveCompletion(reply)) { + throw new InvalidDataException( + "Got non-positive completion for feed $url after $stage: $reply") + } + } +} diff --git a/transmart-data/samples/studies/download_tarball.sh b/transmart-data/samples/studies/download_tarball.sh new file mode 100755 index 000000000..a55c1d5a2 --- /dev/null +++ b/transmart-data/samples/studies/download_tarball.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function main { + local readonly destination="$1" + shift + local readonly urls="$@" + local u= + + for u in $urls; do + echo Trying "$u" + curl -o "$destination" -L -f "$u" + if [[ $? -eq 0 ]]; then + exit 0 + fi + done + + exit 1 +} + +main "$@" diff --git a/transmart-data/samples/studies/enumerate.php b/transmart-data/samples/studies/enumerate.php new file mode 100644 index 000000000..88c2e4893 --- /dev/null +++ b/transmart-data/samples/studies/enumerate.php @@ -0,0 +1,68 @@ +]\n"); + exit(1); +} + +$desired_type = $argc == 3 ? $argv[2] : null; + +$seen_pairs = []; +foreach (new SplFileObject(__DIR__ . '/datasets') as $l) { + if (!trim($l) || $l[0] == '#') { + continue; + } + list($study, $type, $dummy) = preg_split('/\s+/', $l, 3); + if ($desired_type && $desired_type != $type) { + continue; + } + if ($argv[1] == 'tarballs') { + echo "${study}/${study}_${type}.tar.xz\n"; + } elseif ($argv[1] == 'load_targets') { + $target = "load_${type}_$study"; + $seen_pairs[md5($target, true)] = null; + echo "$target\n"; + } +} + +// load stuff that was put directly on samples/studies +if ($argv[1] != 'load_targets') { + return; +} +class DirectoryRecursiveFilterIterator extends RecursiveFilterIterator { + function accept() { + return $this->getDepth() <= 2 && $this->getType() == 'dir'; + } + function hasChildren() { + return parent::hasChildren() && $this->getDepth() < 2; + } + function current() { + return explode('/', $this->getRelativePath()); + } + private function getDepth() { + return substr_count($this->getRelativePath(), '/') + 1; + } + private function getRelativePath() { + return substr($this->key(), strlen(__DIR__) + 1); + } +} + +$it = new RecursiveIteratorIterator( + new DirectoryRecursiveFilterIterator( + new RecursiveDirectoryIterator(__DIR__, + FilesystemIterator::KEY_AS_PATHNAME | + FilesystemIterator::CURRENT_AS_FILEINFO | + FilesystemIterator::SKIP_DOTS | + FilesystemIterator::UNIX_PATHS))); + +foreach ($it as $arr) { + list($study, $type) = $arr; + if ($desired_type && $desired_type != $type) { + continue; + } + $target = "load_${type}_$study"; + if (!key_exists(md5($target, true), $seen_pairs)) { + echo "$target\n"; + } +} diff --git a/transmart-data/samples/studies/get_file_location.php b/transmart-data/samples/studies/get_file_location.php new file mode 100644 index 000000000..1160dcb5a --- /dev/null +++ b/transmart-data/samples/studies/get_file_location.php @@ -0,0 +1,30 @@ +/_.tar.xz\n"); + exit(1); +} + +if (!preg_match('@(.+?)/\1_(.+)\.tar.xz@', $argv[1], $matches)) { + fwrite(STDERR, "Bad argument: $argv[1]\n"); + exit(1); +} + +$desired_study = $matches[1]; +$desired_type = $matches[2]; + +foreach (new SplFileObject('./datasets') as $l) { + if (!trim($l) || $l[0] == '#') { + continue; + } + list($study, $type, $location) = preg_split('/\s+/', $l, 3); + if ($study == $desired_study && $type == $desired_type) { + $mult_locations = preg_split('/\s+/', $location); + shuffle($mult_locations); + echo implode(' ', $mult_locations), "\n"; + exit(0); + } +} + +fprintf(STDERR, "Pair (%s, %s) not found!\n", $desired_study, $desired_type); +exit(1); diff --git a/transmart-data/samples/studies/public-feeds b/transmart-data/samples/studies/public-feeds new file mode 100644 index 000000000..3c74c0fff --- /dev/null +++ b/transmart-data/samples/studies/public-feeds @@ -0,0 +1 @@ +http-index http://library.transmartfoundation.org/datasets/datasets_index diff --git a/transmart-gwas-plugin/grails-app/conf/org/transmartproject/gwas/AuditLogFilters.groovy b/transmart-gwas-plugin/grails-app/conf/org/transmartproject/gwas/AuditLogFilters.groovy deleted file mode 100644 index 947e2f84d..000000000 --- a/transmart-gwas-plugin/grails-app/conf/org/transmartproject/gwas/AuditLogFilters.groovy +++ /dev/null @@ -1,73 +0,0 @@ -package org.transmartproject.gwas - -// from transmart-core-api -import org.transmartproject.core.users.User -// from transmart-extensions -import org.transmart.biomart.BioAssayAnalysis - -class AuditLogFilters { - - def auditLogService - def experimentService - User currentUserBean - - public String getAnalysisName(Long analysisId) { - return analysisId ? BioAssayAnalysis.get(analysisId)?.name : null - } - - public String getAnalysisNames(String analysisIds) { - return analysisIds?.split(",")?.map {getAnalysisName(it.toLong())}?.filter()?.join("|") - } - - def filters = { - search(controller: 'GWAS', action: 'getFacetResults') { - before = { model -> - auditLogService.report("GWAS Active Filter", request, - user: currentUserBean, - query: params.q ?: '', - facetQuery: params.fq ?: '', - ) - } - } - other(controller: 'GWAS|gwas*|uploadData', action: '*', actionExclude:'getFacetResults|newSearch|index|getDynatree|getSearchCategories') { - after = { model -> - def task = "Gwas (${controllerName}.${actionName})" - switch (actionName) { - case "getAnalysisResults": - if (params.export) { - task = "Gwas CSV Export" - } else { - task = "Gwas Analysis Access" - } - break - case "getTrialAnalysis": - task = "Gwas Study Access" - break - case "getTableResults": - task = "Gwas Table View" - break - case "webStartPlotter": - task = "Gwava" - break - case "exportAnalysis": - if (params.isLink == "true") { - task = "Gwas Files Export" - } else { - task = "Gwas Email Analysis" - } - break - } - String analysis = (params?.analysisIds) ? - getAnalysisNames(params.analysisIds) : - getAnalysisName(params.getLong('analysisId')) - auditLogService.report(task, request, - user: currentUserBean, - experiment: experimentService.getExperimentAccession(params.getLong('trialNumber')) ?: '', - analysis: analysis ?: '', - export: params.export ?: '', - ) - } - } - } - -} diff --git a/transmart-gwas-plugin/grails-app/controllers/com/recomdata/grails/plugin/gwas/GwasSearchController.groovy b/transmart-gwas-plugin/grails-app/controllers/com/recomdata/grails/plugin/gwas/GwasSearchController.groovy index 2a6dd234c..db0ed8c4a 100644 --- a/transmart-gwas-plugin/grails-app/controllers/com/recomdata/grails/plugin/gwas/GwasSearchController.groovy +++ b/transmart-gwas-plugin/grails-app/controllers/com/recomdata/grails/plugin/gwas/GwasSearchController.groovy @@ -3,12 +3,11 @@ package com.recomdata.grails.plugin.gwas import au.com.bytecode.opencsv.CSVWriter import com.recomdata.transmart.domain.searchapp.FormLayout import grails.converters.JSON +import grails.web.mapping.LinkGenerator import org.grails.web.json.JSONObject; -import org.grails.web.mapping.LinkGenerator import org.transmart.biomart.BioAssayAnalysis import org.transmart.biomart.Experiment import org.transmart.searchapp.* -import grails.util.Holders import java.lang.reflect.UndeclaredThrowableException import java.util.regex.Matcher @@ -20,6 +19,7 @@ import static java.util.UUID.randomUUID class GwasSearchController { + LinkGenerator grailsLinkGenerator def regionSearchService def gwasWebService def RModulesFileWritingService diff --git a/transmart-gwas-plugin/grails-app/controllers/org/transmartproject/gwas/GwasInterceptor.groovy b/transmart-gwas-plugin/grails-app/controllers/org/transmartproject/gwas/GwasInterceptor.groovy new file mode 100644 index 000000000..eb3de98b0 --- /dev/null +++ b/transmart-gwas-plugin/grails-app/controllers/org/transmartproject/gwas/GwasInterceptor.groovy @@ -0,0 +1,81 @@ +package org.transmartproject.gwas + +// from transmart-core-api +import org.transmartproject.core.users.User +// from transmart-extensions +import org.transmart.biomart.BioAssayAnalysis + +class GwasInterceptor { + + def auditLogService + def experimentService + User currentUserBean + + GwasInterceptor(){ + match(controller: ~/(GWAS|gwas*|uploadData)/).excludes(action: ~/(newSearch|index|getDynatree|getSearchCategories)/) + } + + public String getAnalysisName(Long analysisId) { + return analysisId ? BioAssayAnalysis.get(analysisId)?.name : null + } + + public String getAnalysisNames(String analysisIds) { + return analysisIds?.split(",")?.map {getAnalysisName(it.toLong())}?.filter()?.join("|") + } + + boolean before() { + if (actionName == 'getFacetResults') { + auditLogService.report("GWAS Active Filter", request, + user: currentUserBean, + query: params.q ?: '', + facetQuery: params.fq ?: '', + ) + } + true + } + + boolean after() { + if (actionName != 'getFacetResults') { + def task = "Gwas (${controllerName}.${actionName})" + switch (actionName) { + case "getAnalysisResults": + if (params.export) { + task = "Gwas CSV Export" + } else { + task = "Gwas Analysis Access" + } + break + case "getTrialAnalysis": + task = "Gwas Study Access" + break + case "getTableResults": + task = "Gwas Table View" + break + case "webStartPlotter": + task = "Gwava" + break + case "exportAnalysis": + if (params.isLink == "true") { + task = "Gwas Files Export" + } else { + task = "Gwas Email Analysis" + } + break + } + String analysis = (params?.analysisIds) ? + getAnalysisNames(params.analysisIds) : + getAnalysisName(params.getLong('analysisId')) + auditLogService.report(task, request, + user: currentUserBean, + experiment: experimentService.getExperimentAccession(params.getLong('trialNumber')) ?: '', + analysis: analysis ?: '', + export: params.export ?: '', + ) + } + true + } + + void afterView() { + // no-op + } +} diff --git a/transmart-java/src/main/groovy/com/recomdata/transmart/data/export/ExportDataProcessor.java b/transmart-java/src/main/groovy/com/recomdata/transmart/data/export/ExportDataProcessor.java deleted file mode 100644 index 5613bd49d..000000000 --- a/transmart-java/src/main/groovy/com/recomdata/transmart/data/export/ExportDataProcessor.java +++ /dev/null @@ -1,86 +0,0 @@ -/************************************************************************* - * tranSMART - translational medicine data mart - * - * Copyright 2008-2012 Janssen Research & Development, LLC. - * - * This product includes software developed at Janssen Research & Development, LLC. - * - * This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License - * as published by the Free Software * Foundation, either version 3 of the License, or (at your option) any later version, along with the following terms: - * 1. You may convey a work based on this program in accordance with section 5, provided that you retain the above notices. - * 2. You may convey verbatim copies of this program code as you receive it, in any medium, provided that you retain the above notices. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along with this program. If not, see . - * - * - ******************************************************************/ - - -/** - * - */ -package com.recomdata.transmart.data.export; - -import com.recomdata.transmart.TransmartContextHolder; -import com.recomdata.transmart.data.export.util.FTPUtil; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Map; - -/** - * @author SMunikuntla - * - */ -public class ExportDataProcessor { - - private static org.apache.log4j.Logger log = Logger - .getLogger(ExportDataProcessor.class); - @SuppressWarnings("rawtypes") - private static final Map config = TransmartContextHolder.getGrailsApplication().getFlatConfig(); - - private static final String TEMP_DIR = (String) config.get("com.recomdata.plugins.tempFolderDirectory"); - - public InputStream getExportJobFileStream(String fileToGet) { - InputStream inputStream = null; - File jobZipFile = null; - try { - if (StringUtils.isEmpty(fileToGet)) - return null; - - inputStream = FTPUtil.downloadFile(true, fileToGet); - //If the file was not found at the FTP location try to download it from the server Temp dir - if (null == inputStream) { - String filePath = TEMP_DIR + File.separator + fileToGet; - jobZipFile = new File(filePath); - if (jobZipFile.isFile()) { - inputStream = new FileInputStream(jobZipFile); - } - } - } catch (Exception e) { - log.error("Failed to SFTP GET the ZIP file"); - } - - return inputStream; - } - - @SuppressWarnings("unused") - private String getZipFileName(String studyName) { - StringBuilder fileName = new StringBuilder(); - DateFormat formatter = new SimpleDateFormat("MMddyyyyHHmmss"); - fileName.append(TEMP_DIR); - fileName.append(studyName); - fileName.append(formatter.format(Calendar.getInstance().getTime())); - fileName.append(".zip"); - - return fileName.toString(); - } -} diff --git a/transmart-rest-api/grails-app/conf/org/transmartproject/rest/AuditLogFilters.groovy b/transmart-rest-api/grails-app/conf/org/transmartproject/rest/AuditLogFilters.groovy deleted file mode 100644 index 13f8bdd8e..000000000 --- a/transmart-rest-api/grails-app/conf/org/transmartproject/rest/AuditLogFilters.groovy +++ /dev/null @@ -1,34 +0,0 @@ -package org.transmartproject.rest - -import org.transmartproject.core.users.User - -class AuditLogFilters { - - def accessLogService - User currentUserBean - - def filters = { - lowDim(controller: 'observation', action:'*') { - after = { model -> - def fullUrl = "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}" - def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr - - accessLogService.report(currentUserBean, 'REST API Data Retrieval', - eventMessage: "User (IP: ${ip}) got low dim. data with ${fullUrl}", - requestURL: fullUrl) - } - } - - highDim(controller: 'highDim', action:'*') { - after = { model -> - def fullUrl = "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}" - def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr - - accessLogService.report(currentUserBean, 'REST API Data Retrieval', - eventMessage: "User (IP: ${ip}) got high dim. data with ${fullUrl}", - requestURL: fullUrl) - } - } - } - -} diff --git a/transmart-rest-api/grails-app/controllers/org/transmartproject/interceptors/DataRetrievalInterceptor.groovy b/transmart-rest-api/grails-app/controllers/org/transmartproject/interceptors/DataRetrievalInterceptor.groovy new file mode 100644 index 000000000..3429606d2 --- /dev/null +++ b/transmart-rest-api/grails-app/controllers/org/transmartproject/interceptors/DataRetrievalInterceptor.groovy @@ -0,0 +1,28 @@ +package org.transmartproject.interceptors + +import org.transmartproject.core.users.User + +class DataRetrievalInterceptor { + def accessLogService + User currentUserBean + + DataRetrievalInterceptor(){ + match(controller: ~/(observation|highDim)/) + } + + boolean before() {true} + + boolean after(){ + def fullUrl = "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}" + def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr + + def dataType = controllerName == 'observation' ? 'low dim' : 'high dim' + + accessLogService.report(currentUserBean, 'REST API Data Retrieval', + eventMessage: "User (IP: ${ip}) got ${dataType}. data with ${fullUrl}", + requestURL: fullUrl) + true + } + + void afterView(){} +} diff --git a/transmartApp/build.gradle b/transmartApp/build.gradle index dac384c51..330f416d1 100644 --- a/transmartApp/build.gradle +++ b/transmartApp/build.gradle @@ -125,15 +125,20 @@ dependencies { compile files('lib/GenePattern.jar') compile 'org.jfree:jfreesvg:3.1' compile project(':transmart-core-api') - compile project (':transmart-gwas-plugin') - compile project(':Rmodules') compile project(':transmart-core-db') compile project(':transmart-java') compile project(':search-domain') compile project(':biomart-domain') compile project(':transmart-legacy-db') - compile project(':folder-management-plugin') - compile project(':transmart-gwas-plugin') +} + +grails { + exploded = true + plugins { + compile project(':Rmodules') + compile project(':transmart-gwas-plugin') + compile project(':folder-management-plugin') + } } assets { diff --git a/transmartApp/grails-app/assets/javascripts/advancedWorkflowFunctions.js b/transmartApp/grails-app/assets/javascripts/advancedWorkflowFunctions.js index dd268eb2d..b4a60537f 100644 --- a/transmartApp/grails-app/assets/javascripts/advancedWorkflowFunctions.js +++ b/transmartApp/grails-app/assets/javascripts/advancedWorkflowFunctions.js @@ -445,7 +445,7 @@ function showGeneSelection() } }], autoLoad: { - url: pageInfo.basePath+'/panels/compareGeneSelection.html', + url: pageInfo.basePath+'/assets/panels/compareGeneSelection.html', scripts: true, nocache:true, discardUrl:true, diff --git a/transmartApp/grails-app/assets/javascripts/datasetExplorer/datasetExplorer.js b/transmartApp/grails-app/assets/javascripts/datasetExplorer/datasetExplorer.js index 74dea77c0..cc98f29d8 100644 --- a/transmartApp/grails-app/assets/javascripts/datasetExplorer/datasetExplorer.js +++ b/transmartApp/grails-app/assets/javascripts/datasetExplorer/datasetExplorer.js @@ -864,7 +864,7 @@ Ext.onReady(function () { width: 490, split: false, autoLoad: { - url: pageInfo.basePath+'/panels/setValueDialog.html', + url: pageInfo.basePath+'/assets/panels/setValueDialog.html', scripts: true, nocache: true, discardUrl: true, @@ -1628,7 +1628,7 @@ function showExportStepSplitTimeSeries() { ], resizable: false , autoLoad: { - url: pageInfo.basePath+'/panels/exportStepSplitTimeSeries.html', + url: pageInfo.basePath+'/assets/panels/exportStepSplitTimeSeries.html', scripts: true, nocache: true, discardUrl: true, @@ -1677,7 +1677,7 @@ function showExportStepDataSelection() { ], resizable: false, autoLoad: { - url: pageInfo.basePath+'/panels/exportStepDataSelection.html', + url: pageInfo.basePath+'/assets/panels/exportStepDataSelection.html', scripts: true, nocache: true, discardUrl: true, diff --git a/transmartApp/grails-app/assets/javascripts/datasetExplorer/i2b2common.js b/transmartApp/grails-app/assets/javascripts/datasetExplorer/i2b2common.js index a3562c623..7dfb3cc2d 100644 --- a/transmartApp/grails-app/assets/javascripts/datasetExplorer/i2b2common.js +++ b/transmartApp/grails-app/assets/javascripts/datasetExplorer/i2b2common.js @@ -1442,7 +1442,7 @@ function showCompareStepPathwaySelection() resizable: false, autoLoad: { - url: pageInfo.basePath+'/panels/compareStepPathwaySelection.html', + url: pageInfo.basePath+'/assets/panels/compareStepPathwaySelection.html', scripts: true, nocache:true, discardUrl:true, @@ -1630,7 +1630,7 @@ function showCompareStepPathwaySelectionRBM() resizable: false, autoLoad: { - url: pageInfo.basePath+'/panels/compareStepPathwaySelectionRBM.html', + url: pageInfo.basePath+'/assets/panels/compareStepPathwaySelectionRBM.html', scripts: true, nocache:true, discardUrl:true, diff --git a/transmartApp/src/main/resources/public/panels/compareGeneSelection.html b/transmartApp/grails-app/assets/javascripts/panels/compareGeneSelection.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/compareGeneSelection.html rename to transmartApp/grails-app/assets/javascripts/panels/compareGeneSelection.html diff --git a/transmartApp/src/main/resources/public/panels/compareStepPathwaySelection.html b/transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelection.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/compareStepPathwaySelection.html rename to transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelection.html diff --git a/transmartApp/src/main/resources/public/panels/compareStepPathwaySelectionKMeans.html b/transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelectionKMeans.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/compareStepPathwaySelectionKMeans.html rename to transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelectionKMeans.html diff --git a/transmartApp/src/main/resources/public/panels/compareStepPathwaySelectionRBM.html b/transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelectionRBM.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/compareStepPathwaySelectionRBM.html rename to transmartApp/grails-app/assets/javascripts/panels/compareStepPathwaySelectionRBM.html diff --git a/transmartApp/src/main/resources/public/panels/dataTypesExportDataSelection.html b/transmartApp/grails-app/assets/javascripts/panels/dataTypesExportDataSelection.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/dataTypesExportDataSelection.html rename to transmartApp/grails-app/assets/javascripts/panels/dataTypesExportDataSelection.html diff --git a/transmartApp/src/main/resources/public/panels/exportStepDataSelection.html b/transmartApp/grails-app/assets/javascripts/panels/exportStepDataSelection.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/exportStepDataSelection.html rename to transmartApp/grails-app/assets/javascripts/panels/exportStepDataSelection.html diff --git a/transmartApp/src/main/resources/public/panels/exportStepProgress.html b/transmartApp/grails-app/assets/javascripts/panels/exportStepProgress.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/exportStepProgress.html rename to transmartApp/grails-app/assets/javascripts/panels/exportStepProgress.html diff --git a/transmartApp/src/main/resources/public/panels/exportStepSplitTimeSeries.html b/transmartApp/grails-app/assets/javascripts/panels/exportStepSplitTimeSeries.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/exportStepSplitTimeSeries.html rename to transmartApp/grails-app/assets/javascripts/panels/exportStepSplitTimeSeries.html diff --git a/transmartApp/src/main/resources/public/panels/setValueDialog.html b/transmartApp/grails-app/assets/javascripts/panels/setValueDialog.html similarity index 100% rename from transmartApp/src/main/resources/public/panels/setValueDialog.html rename to transmartApp/grails-app/assets/javascripts/panels/setValueDialog.html diff --git a/transmartApp/grails-app/assets/stylesheets/analysetab.css b/transmartApp/grails-app/assets/stylesheets/analysetab.css index fe445f916..4d77aa6bd 100644 --- a/transmartApp/grails-app/assets/stylesheets/analysetab.css +++ b/transmartApp/grails-app/assets/stylesheets/analysetab.css @@ -1,6 +1,6 @@ /* -*= require jquery/ui +*= require jquery/ui/jquery-ui.min.css *= require jquery-plugins.css *= require extjs.css *= require session_timeout.css diff --git a/transmartApp/grails-app/assets/stylesheets/browseTab.css b/transmartApp/grails-app/assets/stylesheets/browseTab.css index 8856c5108..7d693377a 100644 --- a/transmartApp/grails-app/assets/stylesheets/browseTab.css +++ b/transmartApp/grails-app/assets/stylesheets/browseTab.css @@ -4,7 +4,6 @@ *= require sanofi.css *= require colorbox.css *= require jquery/jqueryDatatable.css -*= require jquery/ui *= require jquery-plugin.css *= require extjs.css *= require session_timeout.css diff --git a/transmartApp/grails-app/assets/stylesheets/datasetExplorer.css b/transmartApp/grails-app/assets/stylesheets/datasetExplorer.css index aeebdcdc8..772059527 100644 --- a/transmartApp/grails-app/assets/stylesheets/datasetExplorer.css +++ b/transmartApp/grails-app/assets/stylesheets/datasetExplorer.css @@ -99,7 +99,7 @@ a.toplink1:hover{ background-image:url(numeric.gif) !important; } .alphaicon{ - background-image:url(images/alpha.gif) !important; + background-image:url(alpha.gif) !important; } .foldernumericicon{ background-image:url(folder_numeric.png) !important; @@ -108,25 +108,25 @@ a.toplink1:hover{ .modifiericon{background-image:url(ONT_MODIFIER_leaf.gif) !important;} .printbutton{background-image:url(printer.png) !important;} .savebutton{ - background-image:url(../images/disk.png) !important; + background-image:url(disk.png) !important; } .progress-spinner { - background: url(../images/spinner.gif) 0px 0px / 16px 16px no-repeat transparent; + background: url(spinner.gif) 0px 0px / 16px 16px no-repeat transparent; display:inline-block; height: 16px; width: 16px; } .progress-error { - background: url(../images/remove.png) 0px 0px / 16px 16px no-repeat transparent; + background: url(remove.png) 0px 0px / 16px 16px no-repeat transparent; display:inline-block; height: 16px; width: 16px; } .progress-completed { - background: url(../images/green_check2.png) 0px 0px / 16px 16px no-repeat transparent; + background: url(green_check2.png) 0px 0px / 16px 16px no-repeat transparent; display:inline-block; height: 16px; width: 16px; @@ -147,7 +147,7 @@ height:0; position:absolute; top:30px; width:100%; - background: #fafafa url(../images/skin/shadow30.jpg); + background: #fafafa url(skin/shadow30.jpg); } @@ -183,7 +183,7 @@ a.button span { .ext-ie .x-form-text {position:static !important;} -.x-tree-node-leaf .x-tree-node-icon{background-image:url(../images/nonaccessible.gif);} +.x-tree-node-leaf .x-tree-node-icon{background-image:url(nonaccessible.gif);} div.analysis { font-family: arial; @@ -243,12 +243,12 @@ div.smalltitle { .contextHelpBtn { - background-image:url(../images/help/helpicon_gray.jpg) !important; + background-image:url(help/helpicon_gray.jpg) !important; color:#ff0000; } .x-tool-sampleExplorerHelpButton { - background-image:url(../images/help/helpicon_gray.jpg) !important; + background-image:url(help/helpicon_gray.jpg) !important; color:#ff0000; } @@ -415,10 +415,10 @@ table.tablesorterAnalysisResults tbody tr.odd td { background-color:#F0F0F6; } table.tablesorterAnalysisResults thead tr .headerSortUp { - background-image: url(../images/asc.gif); + background-image: url(asc.gif); } table.tablesorterAnalysisResults thead tr .headerSortDown { - background-image: url(../images/desc.gif); + background-image: url(desc.gif); } table.tablesorterAnalysisResults thead tr .headerSortDown, table.tablesorter thead tr .headerSortUp { background-color: #8dbdd8; @@ -491,7 +491,7 @@ content: counter(pages); } .x-tool-help { - background-image: url("../images/help/tool-sprites-rdc.gif"); + background-image: url("help/tool-sprites-rdc.gif"); background-position: 0 -300px; } diff --git a/transmartApp/grails-app/assets/stylesheets/jquery-plugin.css b/transmartApp/grails-app/assets/stylesheets/jquery-plugin.css index 84f126363..dd718eadc 100644 --- a/transmartApp/grails-app/assets/stylesheets/jquery-plugin.css +++ b/transmartApp/grails-app/assets/stylesheets/jquery-plugin.css @@ -1,6 +1,6 @@ /* -*= require jquery/ui +*= require jquery/ui/jquery-ui.min.css *= require jquery.loadmask.css *= require jquery/multiselect/ui.multiselect.css' *= require jquery/skin/ui.dynatree.css' diff --git a/transmartApp/grails-app/controllers/GenePatternController.groovy b/transmartApp/grails-app/controllers/GenePatternController.groovy index 3ac50de52..6b348bc42 100644 --- a/transmartApp/grails-app/controllers/GenePatternController.groovy +++ b/transmartApp/grails-app/controllers/GenePatternController.groovy @@ -6,16 +6,17 @@ import com.recomdata.genepattern.JobStatus import com.recomdata.genepattern.WorkflowStatus import grails.converters.JSON import org.json.JSONObject +import org.quartz.JobBuilder import org.quartz.JobDataMap -import org.quartz.JobDetail -import org.quartz.SimpleTrigger +import org.quartz.TriggerBuilder +import org.quartz.core.QuartzScheduler import org.transmart.CohortInformation import org.transmart.ExperimentData import org.transmart.HeatmapValidator import org.transmart.searchapp.AccessLog class GenePatternController { - def quartzScheduler + QuartzScheduler quartzScheduler def genePatternService def springSecurityService def i2b2HelperService @@ -215,13 +216,18 @@ class GenePatternController { jdm.put("userName", userName) def group = "heatmaps" - def jobDetail = new JobDetail(jobName, group, genePatternService.getClass()) - jobDetail.setJobDataMap(jdm) + def jobDetail = JobBuilder.newJob(genePatternService.class) + .withIdentity(jobName, group) + .setJobData(jdm) + .build() if (asyncJobService.updateStatus(jobName, statusList[5])) { return } - def trigger = new SimpleTrigger("triggerNow", group) + def trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity(group) + .build() quartzScheduler.scheduleJob(jobDetail, trigger) ////println "WIP: Gene Pattern replacement" // log.debug('WIP: Gene Pattern replacement') @@ -344,15 +350,20 @@ class GenePatternController { jdm.put("error", error) def group = "heatmaps" - def jobDetail = new JobDetail(jobName, group, genePatternService.getClass()) - jobDetail.setJobDataMap(jdm) + def jobDetail = JobBuilder.newJob(genePatternService.class) + .withIdentity(jobName, group) + .setJobData(jdm) + .build() // if (asyncJobService.updateStatus(jobName, statusList[3])) { return } - def trigger = new SimpleTrigger("triggerNow", group) + def trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity(group) + .build() quartzScheduler.scheduleJob(jobDetail, trigger) // println "WIP: Gene Pattern replacement" // log.debug('WIP: Gene Pattern replacement') @@ -452,11 +463,16 @@ class GenePatternController { jdm.put("userName", userName) def group = "heatmaps" - def jobDetail = new JobDetail(jobName, group, genePatternService.getClass()) - jobDetail.setJobDataMap(jdm) + def jobDetail = JobBuilder.newJob(genePatternService.class) + .withIdentity(jobName, group) + .setJobData(jdm) + .build() asyncJobService.updateStatus(jobName, statusList[3]) - def trigger = new SimpleTrigger("triggerNow", group) + def trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity(group) + .build() quartzScheduler.scheduleJob(jobDetail, trigger) //println "WIP: Gene Pattern replacement" //log.debug('WIP: Gene Pattern replacement') @@ -833,11 +849,16 @@ class GenePatternController { jdm.put("userName", userName) def group = "heatmaps" - def jobDetail = new JobDetail(jobName, group, genePatternService.getClass()) - jobDetail.setJobDataMap(jdm) + def jobDetail = JobBuilder.newJob(genePatternService.class) + .withIdentity(jobName, group) + .setJobData(jdm) + .build() asyncJobService.updateStatus(jobName, statusList[3]) - def trigger = new SimpleTrigger("triggerNow", group) + def trigger = TriggerBuilder.newTrigger() + .startNow() + .withIdentity(group) + .build() quartzScheduler.scheduleJob(jobDetail, trigger) //println "WIP: Gene Pattern replacement" //log.debug('WIP: Gene Pattern replacement') diff --git a/transmartApp/grails-app/controllers/org/transmart/ontology/QueryToolController.groovy b/transmartApp/grails-app/controllers/org/transmart/ontology/QueryToolController.groovy index 10b292000..c97d6918b 100644 --- a/transmartApp/grails-app/controllers/org/transmart/ontology/QueryToolController.groovy +++ b/transmartApp/grails-app/controllers/org/transmart/ontology/QueryToolController.groovy @@ -1,6 +1,7 @@ package org.transmart.ontology import grails.converters.JSON +import grails.transaction.Transactional import org.transmart.marshallers.QueryResultConverter import org.transmartproject.core.exceptions.InvalidRequestException import org.transmartproject.core.querytool.QueryDefinition @@ -18,6 +19,7 @@ class QueryToolController { * * The result is a JSON serialized QueryResult. */ + @Transactional def runQueryFromDefinition() { QueryDefinition definition = queryDefinitionXmlService.fromXml(request.reader) diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ChartInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ChartInterceptor.groovy new file mode 100644 index 000000000..f5034be30 --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ChartInterceptor.groovy @@ -0,0 +1,44 @@ +package org.transmartproject.interceptors + +import org.transmartproject.core.users.User + +class ChartInterceptor { + + def auditLogService + def studyIdService + User currentUserBean + + ChartInterceptor(){ + match(controller: 'chart').excludes(action:~/(clearGrid|displayChart|childConceptPatientCounts)/) + } + + boolean before() { + if (!auditLogService.enabled) return true + def result_instance_id1 = params.result_instance_id1 ?: '' + def result_instance_id2 = params.result_instance_id2 ?: '' + def studies = '' + if (params.concept_key) { + studies = studyIdService.getStudyIdForConceptKey(params.concept_key) ?: '' + } + if (studies.empty) { + studies = studyIdService.getStudyIdsForQueries([result_instance_id1, result_instance_id2]) + } + def task = "Summary Statistics" + if (actionName == "childConceptPatientCounts") { + task = "Clinical Data Access" + } + auditLogService.report(task, request, + study: studies, + user: currentUserBean, + subset1: result_instance_id1, + subset2: result_instance_id2 + ) + true + } + + boolean after() { true } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ConceptsInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ConceptsInterceptor.groovy new file mode 100644 index 000000000..ed644b6ca --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/ConceptsInterceptor.groovy @@ -0,0 +1,37 @@ +package org.transmartproject.interceptors + +import org.transmart.audit.AuditLogService +import org.transmartproject.core.users.User + + +class ConceptsInterceptor { + + AuditLogService auditLogService + def studyIdService + User currentUserBean + + ConceptsInterceptor(){ + match(controller:'concepts', action: 'getChildren') + } + + boolean before() { + if (!auditLogService.enabled) return true + def studies = null + if (params.concept_key) { + studies = studyIdService.getStudyIdForConceptKey(params.concept_key, studyConceptOnly: true) + } + if (studies == null) return true + def task = "Clinical Data Access" + auditLogService.report(task, request, + study: studies, + user: currentUserBean + ) + true + } + + boolean after() { true } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/DataExportInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/DataExportInterceptor.groovy new file mode 100644 index 000000000..20180f4c6 --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/DataExportInterceptor.groovy @@ -0,0 +1,77 @@ +package org.transmartproject.interceptors + +import grails.converters.JSON +import org.transmartproject.core.log.AccessLogEntryResource +import org.transmartproject.core.users.User + + + +class DataExportInterceptor { + + AccessLogEntryResource accessLogService + def auditLogService + def studyIdService + User currentUserBean + + DataExportInterceptor(){ + match(controller: 'dataExport', action:~/(runDataExport|downloadFile)/) + } + /** + * Get export types from the selectedSubsetDataTypeFiles + * field in the parameters. + * Returns a list containing 'LDD' if the parameter contains data type 'CLINICAL' + * and 'HDD' if any other data type is in the parameter. + * + * @param params + * @return a list containing 'LDD', 'HDD', or both. + */ + private List getExportTypes(params) { + def param = params.selectedSubsetDataTypeFiles + def dataFiles = param instanceof String ? + [JSON.parse(param)] : + param.collect { JSON.parse(it) } + Set dataTypes = dataFiles*.dataTypeId as Set + List exportTypes = [] + if ('CLINICAL' in dataTypes) { + exportTypes += 'LDD' + } + if ((dataTypes - 'CLINICAL').size() > 0) { + exportTypes += 'HDD' + } + exportTypes + } + + boolean before() { true } + + boolean after() { + if (actionName == "runDataExport") + { + def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr + def fullUrl = "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}" + def result_instance_id1 = params.result_instance_id1 ?: '' + def result_instance_id2 = params.result_instance_id2 ?: '' + def studies = studyIdService.getStudyIdsForQueries([result_instance_id1, result_instance_id2]) + def exportTypes = getExportTypes(params).join('+') + + accessLogService.report(currentUserBean, 'Data Export', + eventMessage: "User (IP: ${ip}) requested export of data. Http request parameters: ${params}", + requestURL: fullUrl) + auditLogService.report("Clinical Data Exported - ${exportTypes}", request, + study: studies, + user: currentUserBean + ) + } + else if (actionName == "runDataExport") + { + def ip = request.getHeader('X-FORWARDED-FOR') ?: request.remoteAddr + accessLogService.report(currentUserBean, 'Data Export', + eventMessage: "User (IP: ${ip}) downloaded an exported file.", + requestURL: "${request.forwardURI}${request.queryString ? '?' + request.queryString : ''}") + } + true + } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/OauthInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/OauthInterceptor.groovy new file mode 100644 index 000000000..1ab2403e3 --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/OauthInterceptor.groovy @@ -0,0 +1,22 @@ +package org.transmartproject.interceptors + +import org.transmartproject.core.users.User + + +class OauthInterceptor { + //matches OauthController and all actions of controller by naming convention + def auditLogService + User currentUserBean + boolean before() { + auditLogService.report("OAuth authentication", request, + user: currentUserBean, + ) + true + } + + boolean after() { true } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/RWGInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/RWGInterceptor.groovy new file mode 100644 index 000000000..b511b617b --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/RWGInterceptor.groovy @@ -0,0 +1,25 @@ +package org.transmartproject.interceptors + +import org.transmartproject.core.users.User + + +class RWGInterceptor { + def auditLogService + User currentUserBean + RWGInterceptor(){ + match(controller: 'RWG', action: 'getFacetResults') + } + boolean before() { + auditLogService.report("Clinical Data Active Filter", request, + query: params.searchTerms, + user: currentUserBean, + ) + true + } + + boolean after() { true } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/controllers/org/transmartproject/interceptors/UserLandingInterceptor.groovy b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/UserLandingInterceptor.groovy new file mode 100644 index 000000000..e878ed985 --- /dev/null +++ b/transmartApp/grails-app/controllers/org/transmartproject/interceptors/UserLandingInterceptor.groovy @@ -0,0 +1,26 @@ +package org.transmartproject.interceptors + +import org.transmartproject.core.users.User + + +class UserLandingInterceptor { + def auditLogService + User currentUserBean + UserLandingInterceptor(){ + + match(controller: 'userLanding').excludes(action: 'checkHeartBeat') + } + + boolean before() { + auditLogService.report("User Access", request, + user: currentUserBean, + ) + true + } + + boolean after() { true } + + void afterView() { + // no-op + } +} diff --git a/transmartApp/grails-app/services/I2b2HelperService.groovy b/transmartApp/grails-app/services/I2b2HelperService.groovy index ee438cb15..fa7fba37e 100644 --- a/transmartApp/grails-app/services/I2b2HelperService.groovy +++ b/transmartApp/grails-app/services/I2b2HelperService.groovy @@ -13,6 +13,7 @@ import org.transmartproject.db.i2b2data.ConceptDimension import org.transmartproject.db.i2b2data.ObservationFact import org.transmartproject.db.querytool.QtPatientSetCollection import org.transmartproject.db.ontology.AcrossTrialsOntologyTerm +import org.transmartproject.db.util.StringUtils import org.w3c.dom.Document import org.w3c.dom.Node import org.w3c.dom.NodeList @@ -632,7 +633,7 @@ class I2b2HelperService { ORDER BY C_FULLNAME """ log.trace(sqlt); - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> results.put(row[0], getObservationCountForConceptForSubset("\\blah" + row[1], result_instance_id)); }); } @@ -706,9 +707,9 @@ class I2b2HelperService { ON i.c_name=m.c_name ORDER BY c_name"""; sql.eachRow(sqlt, [ - fullname.asLikeLiteral() + "%", // Note: .asLikeLiteral() defined in github: 994dc5bb50055f8b800045f65c8e565b4aa0c113 + StringUtils.asLikeLiteral(fullname) + "%", i, - fullname.asLikeLiteral() + "%", + StringUtils.asLikeLiteral(fullname) + "%", i, result_instance_id ], { row -> @@ -750,7 +751,7 @@ class I2b2HelperService { String sqlt = "SELECT DISTINCT c_name, c_fullname FROM i2b2metadata.i2b2 WHERE C_FULLNAME LIKE ? escape '\\' AND c_hlevel = ? ORDER BY C_FULLNAME"; log.trace(sqlt); - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> if (results.get(row[0]) == null) { results.put(row[0], getObservationCountForConceptForSubset("\\blah" + row[1], result_instance_id)); } else { @@ -763,7 +764,7 @@ class I2b2HelperService { Sql sql = new Sql(dataSource); String sqlt = "SELECT DISTINCT c_name, c_fullname FROM i2b2metadata.i2b2 WHERE C_FULLNAME LIKE ? escape '\\' AND c_hlevel = ? ORDER BY C_FULLNAME"; log.trace(sqlt); - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> results.put(row[0], getObservationCountForConceptForSubset("\\blah" + row[1], result_instance_id)); }); } @@ -784,7 +785,7 @@ class I2b2HelperService { int i = getLevelFromKey(concept_key) + 1; Sql sql = new Sql(dataSource); String sqlt = "SELECT C_FULLNAME, C_METADATAXML FROM i2b2metadata.i2b2 WHERE C_FULLNAME LIKE ? escape '\\' AND c_hlevel = ? ORDER BY C_FULLNAME"; - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> String conceptkey = prefix + row.c_fullname; xml = clobToString(row.c_metadataxml); log.trace("METADATA XML:" + xml); @@ -822,7 +823,7 @@ class I2b2HelperService { FROM i2b2demodata.observation_fact WHERE (((concept_cd IN (select concept_cd from i2b2demodata.concept_dimension c where concept_path LIKE ? escape '\\'))))"""; - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%"], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%"], { row -> i = row[1]; }) return i; @@ -998,7 +999,7 @@ class I2b2HelperService { log.trace("Getting observation count for concept:" + concept_key + " and instance:" + result_instance_id); String fullname = concept_key.substring(concept_key.indexOf("\\", 2), concept_key.length()); - String fullnameLike = fullname.asLikeLiteral() + "%" // Note: .asLikeLiteral() defined in github: 994dc5bb50055f8b800045f65c8e565b4aa0c113 + String fullnameLike = StringUtils.asLikeLiteral(fullname) + "%" // Note: .asLikeLiteral() defined in github: 994dc5bb50055f8b800045f65c8e565b4aa0c113 int i = 0; log.trace("sql inputs: fullnameLike = " + fullnameLike) log.trace("\tresult_instance_id = " + result_instance_id) @@ -5263,7 +5264,7 @@ class I2b2HelperService { int i = getLevelFromKey(concept_key) + 1; Sql sql = new Sql(dataSource); String sqlt = "SELECT C_FULLNAME FROM i2b2metadata.i2b2 WHERE C_FULLNAME LIKE ? escape '\\' AND c_hlevel = ? ORDER BY C_FULLNAME"; - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> String conceptkey = prefix + row.c_fullname; ls.add(keyToPath(conceptkey)); }) @@ -5633,7 +5634,7 @@ class I2b2HelperService { int i = getLevelFromKey(concept_key) + 1; Sql sql = new Sql(dataSource) String sqlt = "SELECT C_FULLNAME, SECURE_OBJ_TOKEN FROM i2b2metadata.i2b2_SECURE WHERE C_FULLNAME LIKE ? escape '\\' AND c_hlevel = ? ORDER BY C_FULLNAME"; - sql.eachRow(sqlt, [fullname.asLikeLiteral() + "%", i], { row -> + sql.eachRow(sqlt, [StringUtils.asLikeLiteral(fullname) + "%", i], { row -> String conceptkey = prefix + row.c_fullname; ls.put(keyToPath(conceptkey), row.secure_obj_token); log.trace("@@found" + conceptkey); diff --git a/transmartApp/grails-app/services/com/recomdata/transmart/data/export/DataExportService.groovy b/transmartApp/grails-app/services/com/recomdata/transmart/data/export/DataExportService.groovy index cec7f2644..ab586ccaa 100644 --- a/transmartApp/grails-app/services/com/recomdata/transmart/data/export/DataExportService.groovy +++ b/transmartApp/grails-app/services/com/recomdata/transmart/data/export/DataExportService.groovy @@ -2,18 +2,17 @@ package com.recomdata.transmart.data.export import com.recomdata.snp.SnpData import com.recomdata.transmart.data.export.exception.DataNotFoundException +import grails.transaction.Transactional import groovy.json.JsonSlurper import org.apache.commons.lang.StringUtils -import org.springframework.transaction.annotation.Transactional import org.transmartproject.core.ontology.Study import org.transmartproject.core.users.User import static org.transmartproject.core.users.ProtectedOperation.WellKnownOperations.EXPORT +@Transactional class DataExportService { - boolean transactional = true - def i2b2ExportHelperService def grailsApplication def snpDataService diff --git a/transmartApp/grails-app/services/com/recomdata/transmart/data/export/ExportService.groovy b/transmartApp/grails-app/services/com/recomdata/transmart/data/export/ExportService.groovy index 7a3a30621..39b3479db 100644 --- a/transmartApp/grails-app/services/com/recomdata/transmart/data/export/ExportService.groovy +++ b/transmartApp/grails-app/services/com/recomdata/transmart/data/export/ExportService.groovy @@ -6,9 +6,10 @@ import com.recomdata.transmart.validate.RequestValidator import grails.converters.JSON import org.apache.commons.lang.StringUtils import org.json.JSONObject +import org.quartz.JobBuilder import org.quartz.JobDataMap import org.quartz.JobDetail -import org.quartz.SimpleTrigger +import org.quartz.TriggerBuilder import org.transmart.authorization.CurrentUserBeanProxyFactory import org.transmart.searchapp.AccessLog @@ -16,8 +17,6 @@ import javax.annotation.Resource class ExportService { - static transactional = true - def i2b2HelperService def i2b2ExportHelperService def jobResultsService @@ -198,13 +197,20 @@ class ExportService { jdm.put("userInContext", currentUserBean.targetSource.target) - def jobDetail = new JobDetail(params.jobName, params.analysis, GenericJobExecutor.class) - jobDetail.setJobDataMap(jdm) + JobDetail jobDetail = JobBuilder.newJob(GenericJobExecutor.class) + .withIdentity(params.jobName, params.analysis) + .setJobData(jdm) + .build() if (asyncJobService.updateStatus(params.jobName, statusList[2])) { return } - def trigger = new SimpleTrigger("triggerNow" + Math.random(), params.analysis) + def randomDelay = Math.random()*10 as int + def startTime = new Date(new Date().time + randomDelay) + def trigger = TriggerBuilder.newTrigger() + .startAt(startTime) + .withIdentity(params.analysis) + .build() quartzScheduler.scheduleJob(jobDetail, trigger) } diff --git a/transmartApp/grails-app/views/chart/_generalOverview.gsp b/transmartApp/grails-app/views/chart/_generalOverview.gsp index 95f372073..887ea4981 100644 --- a/transmartApp/grails-app/views/chart/_generalOverview.gsp +++ b/transmartApp/grails-app/views/chart/_generalOverview.gsp @@ -10,12 +10,12 @@

- ${subsets[1].query} + ${raw(subsets[1].query)}



- ${subsets[2].query} + ${raw(subsets[2].query)}

diff --git a/transmartApp/grails-app/views/chart/_subsetCharts.gsp b/transmartApp/grails-app/views/chart/_subsetCharts.gsp index d11c9d058..d565f456b 100644 --- a/transmartApp/grails-app/views/chart/_subsetCharts.gsp +++ b/transmartApp/grails-app/views/chart/_subsetCharts.gsp @@ -1,12 +1,12 @@ - ${subsets[1]."${prefix}Pie"} + ${raw(subsets[1]."${prefix}Pie")} - ${subsets[2]."${prefix}Pie"} + ${raw(subsets[2]."${prefix}Pie")} \ No newline at end of file diff --git a/transmartApp/grails-app/views/chart/_traditionalComparison.gsp b/transmartApp/grails-app/views/chart/_traditionalComparison.gsp index 924268461..a71d60e8a 100644 --- a/transmartApp/grails-app/views/chart/_traditionalComparison.gsp +++ b/transmartApp/grails-app/views/chart/_traditionalComparison.gsp @@ -4,12 +4,12 @@ %{-- This is hardcoded badness. Multiple (>2) cohort selection should work on that --}% - ${subsets[1]?.conceptBar ?: ''} + ${raw(subsets[1]?.conceptBar ?: '')} - ${subsets[2]?.conceptBar ?: ''} + ${raw(subsets[2]?.conceptBar ?: '')} diff --git a/transmartApp/grails-app/views/chart/_valueComparison.gsp b/transmartApp/grails-app/views/chart/_valueComparison.gsp index a4971e9b8..75b1d9c98 100644 --- a/transmartApp/grails-app/views/chart/_valueComparison.gsp +++ b/transmartApp/grails-app/views/chart/_valueComparison.gsp @@ -4,7 +4,7 @@ - ${subsets?.commons?."${prefix}Histo" ?: ''} + ${raw(subsets?.commons?."${prefix}Histo" ?: '')} @@ -36,7 +36,7 @@ - ${subsets?.commons?."${prefix}Plot" ?: ''} + ${raw(subsets?.commons?."${prefix}Plot" ?: '')} diff --git a/transmartApp/grails-app/views/datasetExplorer/_queryPanel.gsp b/transmartApp/grails-app/views/datasetExplorer/_queryPanel.gsp index 69769d9fd..7c709b597 100644 --- a/transmartApp/grails-app/views/datasetExplorer/_queryPanel.gsp +++ b/transmartApp/grails-app/views/datasetExplorer/_queryPanel.gsp @@ -1,3 +1,7 @@ + + + + @@ -78,3 +82,5 @@ applySubsets(GLOBAL.restoreSubsetId); } + + \ No newline at end of file diff --git a/transmartApp/grails-app/views/datasetExplorer/datasetExplorer.gsp b/transmartApp/grails-app/views/datasetExplorer/datasetExplorer.gsp index d37213a9a..676cf3924 100644 --- a/transmartApp/grails-app/views/datasetExplorer/datasetExplorer.gsp +++ b/transmartApp/grails-app/views/datasetExplorer/datasetExplorer.gsp @@ -14,13 +14,17 @@ <%-- We do not have a central template, so this only works in the database explorer for now --%> - + %{----}% + + + + - - - - - %{----}% diff --git a/transmartApp/grails-app/views/expressionProfile/printView.gsp b/transmartApp/grails-app/views/expressionProfile/printView.gsp index 6db504891..4f7ef6564 100644 --- a/transmartApp/grails-app/views/expressionProfile/printView.gsp +++ b/transmartApp/grails-app/views/expressionProfile/printView.gsp @@ -4,7 +4,7 @@
Subset 1