initial merge of master into openshift branch

.gitignore

@@ -14,7 +14,7 @@
 **/*~
 **/.DS_Store
 
-data/licenses
+data/licenses/*
 data/testfiles/emols_order*
 data/testfiles/all-sdf.sdf.gz
 data/testfiles/chembl_*.sdf.gz
@@ -48,26 +48,29 @@ components/security-keycloak/out
 docker/**/*.war
 docker/xwiki-site/yyy.json
 docker/docker-services
+docker/resources
 
-docker/deploy/data
+docker/deploy/data/pgdata
+docker/deploy/data/docker-services/*
 
 docker/deploy/setenv.sh
 docker/deploy/images/yyy.json
 docker/deploy/images/squonk-realm.json
 
+docker/deploy/images/chemservices/license.cxl
+
 docker/deploy/images/cellexecutor/logging.properties
 
 docker/deploy/images/portal/server.xml
-docker/deploy/images/portal/persistence.properties
 docker/deploy/images/portal/logging.properties
 
 docker/deploy/images/nginx/certs
 docker/deploy/images/nginx/cert.crt
 docker/deploy/images/nginx/cert.key
 docker/deploy/images/nginx/default.ssl.conf
 docker/deploy/images/nginx/default.site.conf
-docker/deploy/images/nginx/sites/squonk.it/html
-docker/deploy/images/nginx/sites/informaticsmatters.com/html
+docker/deploy/images/nginx/sites/squonk.it/*
+docker/deploy/images/nginx/sites/informaticsmatters.com/*
 
 docker/deploy/images/keycloak/keycloak.jks
 

README.md

@@ -43,7 +43,8 @@ For some of the tests and some of the RDKit services you will need RDKit with Ja
 See [here](http://rdkit.org/docs/Install.html#building-from-source) for how to do this.
 If you don't have this then make sure the RDBASE environment variable is not set.
 
-For the ChemAxon services and tests you will need a ChemAxon license file.
+For the ChemAxon services and tests you will need a ChemAxon license file named license.cxl.
+Place this in your $HOME/.chemaxon directory AND in the docker/deploy/images/chemservices dir in the squonk distro.
 
 If for these reasons you find some of the tests failing try using:
 

StructureDatabases.md

@@ -1,6 +1,6 @@
 # Database loaders
 
-Squonk has a chemical database powered by the RDkit cartridge.
+Squonk has a chemical database powered by the RDKit cartridge.
 To use this you need to load datasets into database.
 Currently there is support for
 * eMolecules (bulidng blocks and screening compounds)
@@ -9,7 +9,8 @@ Currently there is support for
 * PDB ligands
 * Chemspace
 
-The code for the loaders is in the rdkit-databases module.
+The code and configuration for the loaders is in the rdkit-lib module.
+At some stage it may be broken out into a separate module.
 
 ## Loading data
 
@@ -41,6 +42,8 @@ Each loader is pre-configured with sensible defaults, but you can override these
 * LIMIT - the number of records to load (default value is 0 which meeans load all records)
 * REPORTING_CHUNK - the frequency to report loading (default apporpopriate to the typical size of the dataset)
 
+For testing set the loadOnly property to restrict the number of structures to load. Reset this to zero to load the 
+entire dataset.
 
 ### Running a loader
 
@@ -80,13 +83,47 @@ To prevent lost connections to the server terminating the load you might want to
 ### Configuring the search service
 
 The searchsearvice needs to know what database tables have been loaded.
-This is currently done using the STRUCTURE_DATABASE_TABLES environment variable
+This is currently done using the CHEMCENTRAL_DATABASE_TABLES environment variable
 that needs to be passes to the chemservices container. To do this set this variable
-in your docker/deply/setenv.sh file. The value must be a colon separated list of table names
+in your docker/deploy/setenv.sh file. The value must be a colon separated list of tabe names
 with no spaces or other characters allowed. For instance:
 
 ```
-export STRUCTURE_DATABASE_TABLES=emolecules_order_sc:emolecules_order_bb:chembl_23:pdb_ligand
+export CHEMCENTRAL_DATABASE_TABLES=emolecules_order_sc:emolecules_order_bb:chembl_23:pdb_ligand
 ```
 
 
+## SQL
+
+The SQL used to create the indexes looks like this (for chembl_23):
+
+```
+DROP TABLE IF EXISTS vendordbs.chembl_23_molfps;
+SELECT * INTO vendordbs.chembl_23_molfps FROM (SELECT id,mol_from_ctab(structure::cstring) m FROM vendordbs.chembl_23) tmp where m IS NOT NULL;
+ALTER TABLE vendordbs.chembl_23_molfps ADD PRIMARY KEY (id);
+ALTER TABLE vendordbs.chembl_23_molfps ADD CONSTRAINT fk_chembl_23_molfps_id FOREIGN KEY (id) REFERENCES vendordbs.chembl_23 (id);
+CREATE INDEX idx_chembl_23_molfps_m ON vendordbs.chembl_23_molfps USING gist(m);
+ALTER TABLE vendordbs.chembl_23_molfps DROP COLUMN IF EXISTS rdk CASCADE;
+ALTER TABLE vendordbs.chembl_23_molfps ADD COLUMN rdk bfp;
+UPDATE vendordbs.chembl_23_molfps SET rdk = rdkit_fp(m);
+CREATE INDEX idx_chembl_23_molfps_rdk ON vendordbs.chembl_23_molfps USING gist(rdk);
+ALTER TABLE vendordbs.chembl_23_molfps DROP COLUMN IF EXISTS mfp2 CASCADE;
+ALTER TABLE vendordbs.chembl_23_molfps ADD COLUMN mfp2 bfp;
+UPDATE vendordbs.chembl_23_molfps SET mfp2 = morganbv_fp(m,2);
+CREATE INDEX idx_chembl_23_molfps_mfp2 ON vendordbs.chembl_23_molfps USING gist(mfp2);
+ALTER TABLE vendordbs.chembl_23_molfps DROP COLUMN IF EXISTS ffp2 CASCADE;
+ALTER TABLE vendordbs.chembl_23_molfps ADD COLUMN ffp2 bfp;
+UPDATE vendordbs.chembl_23_molfps SET ffp2 = featmorganbv_fp(m,2);
+CREATE INDEX idx_chembl_23_molfps_ffp2 ON vendordbs.chembl_23_molfps USING gist(ffp2);
+```
+
+To test substructure search:
+```
+select count(*) from vendordbs.chembl_23_molfps WHERE m@>'c1cccc2c1CNCCN2';
+```
+
+To test similarity search:
+```
+select count(*) from vendordbs.chembl_23_molfps WHERE mfp2%morganbv_fp('CN1CCc2cccc3c2[C@H]1Cc1ccc(CO)c(O)c1-3');
+```
+

XWiki.md

@@ -0,0 +1,36 @@
+# Wiki Setup
+
+When setting up a new Squonk site you often want to copy the Wiki docs to the new site.
+This is a skeleton for doing this.
+TODO - run through the process and document more completely.
+
+## Setup
+
+* In Keycloak in the new site make sure that the following roles exist in the realm:
+** XWiki.XWikiAllGroup
+** XWiki.XWikiAdminGroup
+** XWiki.XWikiUserGroup
+* Add XWiki.XWikiUserGroup as a default role for new users.
+* Update existing users to have this role.
+* Create a user tdudgeon that will own most of the wiki pages (not sure if this is needed).
+
+## Exporting
+
+* On the public Squonk site log into XWiki as a user with XWiki.XWikiAdminGroup role.
+* Click on 'Administer Wiki' in the menu in the top right corner.
+* Go to Export in the Content section.
+* Give the export package a name, and check the 'With history' and 'Backup package' options. 
+* Run the export
+
+An .xar file with the Wiki contents will be downloaded to your computer.
+
+## Importing
+
+* On the new site connect to the Wiki. If this is for the first time XWiki will be initialised.
+* Once done login as a user with XWiki.XWikiAdminGroup role.
+* Click on Administer Wiki in the menu in the top right corner.
+* Go to Import in the Content section.
+* Select the .xar file to import and upload.
+* Select what is to be imported (everything).
+* Run the import. This will take a few mins.
+

components/build.gradle

@@ -52,7 +52,7 @@ allprojects {
     // look at the jackson2-version property here (with correct tag) to get the right Jackson version
     // https://github.com/apache/camel/blob/master/parent/pom.xml
     project.ext.set('jacksonVersion', '2.8.4')
-    project.ext.set('cdkVersion', '2.0')
+    project.ext.set('cdkVersion', '2.1.1')
     project.ext.set('groovyVersion', '2.4.4')
     project.ext.set('weldVersion', '2.3.2.Final')
     project.ext.set('postgresDriverVersion', '9.4-1201-jdbc41')
@@ -61,7 +61,7 @@ allprojects {
     // Define the tag label for docker images.
     // It's the Git branch name or 'latest' if building from 'master'
     def dockerImageTag = grgit.branch.getCurrent().name
-    if (dockerImageTag == 'master') {
+    if (dockerImageTag == 'master' || dockerImageTag == 'openshift-merge') {
         dockerImageTag = 'latest'
     }
     project.ext.set('dockerImageTag', dockerImageTag)
@@ -134,8 +134,6 @@ subprojects {
     }
 }
 
-def hasChemAxonLicense = file('../data/licenses/license.cxl').exists()
-
 task wrapper(type: Wrapper) {
     gradleVersion = '3.2.1'
 }
@@ -160,11 +158,11 @@ task chemServicesWars(type: Copy) {
             tasks.getByPath(':chem-services-cdk-basic:war'),
             tasks.getByPath(':chem-services-chemaxon-basic:war'),
             tasks.getByPath(':chem-services-rdkit-basic:war'),
+            tasks.getByPath(':chem-services-rdkit-search:war'),
             tasks.getByPath(':chem-services-openchemlib-basic:war'),
             tasks.getByPath(':chem-services-smartcyp:war')
     ]
 
-    from "../data/licenses/license.cxl"
     from "../docker/deploy/images/chemservices/chemaxon_reaction_library.zip"
     from("chem-services-cdk-basic/build/libs") {
         include '*.war'
@@ -186,50 +184,40 @@ task chemServicesWars(type: Copy) {
         include '*.war'
         rename '.*', 'chem-services-smartcyp.war'
     }
-    into 'build/chemservices-basic'
+    into 'build/chem-services-basic'
 
 }
 
-task buildChemServicesDockerfile(type: Dockerfile) {
+task dockerFileChemServices(type: Dockerfile) {
 
-    description = 'build the Dockerfile for the chemservices image'
-    destFile = project.file('build/chemservices-basic/Dockerfile')
+    destFile = project.file('build/chem-services-basic/Dockerfile')
     // if changing the version you must also update the hardcoded version number in org.squonk.camel.rdkit.processor.RDKitMoleculeProcessor
-    from "informaticsmatters/rdkit_java_tomcat:Release_2017_03_1"
+    from "informaticsmatters/rdkit-tomcat-debian:Release_2017_09_2"
     maintainer 'Tim Dudgeon <tdudgeon@informaticsmatters.com>'
 
-    runCommand("rm -rf /usr/local/tomcat/webapps/*")
-
     addFile('*.war', "/usr/local/tomcat/webapps/")
-    addFile('license.cxl', 'license.cxl')
     addFile('chemaxon_reaction_library.zip', '/chemaxon_reaction_library.zip')
 
-    environmentVariable("CHEMAXON_LICENSE_URL", "/usr/local/tomcat/license.cxl")
-
     exposePort(8080)
 }
 
-task buildChemServicesDockerImage(type: DockerBuildImage) {
+task dockerImageChemServices(type: DockerBuildImage) {
 
     description = 'Builds the ChemServices Docker Image'
 
-    dependsOn = [chemServicesWars, buildChemServicesDockerfile]
+    dependsOn = [chemServicesWars, dockerFileChemServices]
 
-    inputDir = buildChemServicesDockerfile.destFile.parentFile
+    inputDir = dockerFileChemServices.destFile.parentFile
     tag = "squonk/chemservices-basic:${dockerImageTag}"
 }
 
-task buildDockerImages() {
-
-    description = 'Builds the chemcentral-search, chemservices coreservices, cellexecutor and flyway Docker images'
+task dockerBuildImages() {
 
     dependsOn = [
-            buildChemServicesDockerImage,                  // chemservices-basic image
-            'chem-services-rdkit-search:buildDockerImage', // chemcentral postgres image
-            'rdkit-databases:dockerBuildImage',            // database loader for chemcentral
-            'core-services-server:buildDockerImage',       // coreservices image
-            'cell-executor:dockerBuildImage',              // cellexecutor image
-            'database:buildDockerImage'                    // flyway database migrator
+        'dockerImageChemServices',
+        'core-services-server:buildDockerImage',
+        'cell-executor:dockerBuildImage',
+        'database:buildDockerImage'
     ]
 }
 
@@ -252,7 +240,7 @@ To run the integration tests run the integrationTest task.
 
  */
 task integrationTestInit() {
-    dependsOn = [uploadArchives, buildDockerImages]
+    dependsOn = [uploadArchives, dockerBuildImages]
 }
 
 task integrationTestEnvPrepare(type: Exec) {

components/chem-services-cdk-basic/src/main/java/org/squonk/cdk/services/CamelLifeCycle.java

@@ -16,6 +16,7 @@
 
 package org.squonk.cdk.services;
 
+import org.openscience.cdk.CDK;
 import org.squonk.camel.CamelCommonConstants;
 import java.util.logging.Logger;
 import org.apache.camel.builder.ThreadPoolProfileBuilder;
@@ -56,6 +57,7 @@ public void afterStop(ServletCamelContext scc, SimpleRegistry r) throws Exceptio
     public void beforeAddRoutes(ServletCamelContext scc, SimpleRegistry r) throws Exception {
         LOG.fine("CamelLifeCycle.beforeAddRoutes()");
         LOG.fine("Creating custom thread pool profile named " + CamelCommonConstants.CUSTOM_THREAD_POOL_NAME);
+        LOG.info("Using CDK version " + CDK.getVersion());
         ThreadPoolProfile profile = new ThreadPoolProfileBuilder(CamelCommonConstants.CUSTOM_THREAD_POOL_NAME).poolSize(4).maxPoolSize(50).build();
         scc.getExecutorServiceManager().registerThreadPoolProfile(profile);
     }

components/chem-services-cdk-basic/src/main/java/org/squonk/cdk/services/CdkBasicServices.java

@@ -17,7 +17,6 @@
 package org.squonk.cdk.services;
 
 import org.squonk.dataset.ThinDescriptor;
-import org.squonk.io.IODescriptor;
 import org.squonk.io.IODescriptors;
 import org.squonk.core.HttpServiceDescriptor;
 import org.squonk.execution.steps.StepDefinitionConstants;
@@ -36,7 +35,7 @@ public class CdkBasicServices {
             "https://squonk.it/xwiki/bin/view/Cell+Directory/Data/Verify+structure+%28CDK%29",
             "icons/properties_add.png",
             "verify",
-            new OptionDescriptor[]{OptionDescriptor.FILTER_MODE},
+            new OptionDescriptor[]{OptionDescriptor.FILTER_MODE_PASS},
             ThinDescriptor.DEFAULT_FILTERING_THIN_DESCRIPTOR);
 
     static final HttpServiceDescriptor SERVICE_DESCRIPTOR_LOGP = createServiceDescriptor(