Merge branch 'master' into wtsi_portal

* master: (36 commits)
  Bugfix for the WTSI pheno tests.
  Cleaning some of the custom sorts to remove complexities and duplication.
  Extend the wtsi-pheno testing.
  Fix to make the test coverage reporting work.
  Bugfix - the publish_graph custom sort re-work I did yesterday introduced a couple of problems for non-MGP images/reports.
  Cover more of the codebase with metric_fu.
  Forgot to hit save on the last change... :(
  Small code cleanup for the mgp_graphs data sort.
  Updated the MGP pheno data sorts to group the images (on the detailed report pages) by the protocol description as there can be multiple protocols used per heatmap group. closes #4389
  Update the biomart gem to the latest version.
  Updated the MGP spreadsheet generator to cope with the new 'Test abandoned' status. closes #4257
  Updated the heatmap css to include a grey strike-through for an abandoned test. refs #4257
  Updates to the MGP Pheno heatmap legend. refs #4257
  Small change to stop warnings being fired.
  Rename the image templates so they better reflect their new purpose. refs #4258
  First cut display for the skin screen test reports. refs #4258
  Semantic updates to some CSS class names. refs #4258
  Update the .gitignore file to ignore artefacts from the test suite.
  Modified the WTSI MGP dataset grabs to also get skin-screen data. refs #4258
  Disable the mutagenesis prediction software for Regeneron and mirKO projects until it's set-up to deal with them.
  ...

.gitignore

@@ -8,9 +8,13 @@ doc*
 lib/martsearch/server/public/css/martsearch-*.css
 lib/martsearch/server/public/js/martsearch-base-*.js
 lib/martsearch/server/public/js/martsearch-head-*.js
+pheno_heatmap.xls
+public/css/martsearch-*
+public/js/martsearch-base-*
+public/js/martsearch-head-*
 schema.xml
 simplecov/*
 test/vcr_cassettes*
+tmp/*.json
 tmp/index_builder*
 tmp/pheno_abr*
-pheno_heatmap.xls

.metrics

@@ -0,0 +1,9 @@
+MetricFu::Configuration.run do |config|
+  config.metrics.delete(:flog)
+  config.metrics.delete(:rcov)
+
+  config.flay     = { :dirs_to_flay  => ["lib","config/server/datasets"] }
+  config.reek     = { :dirs_to_reek  => ["lib","config/server/datasets"] }
+  config.roodi    = { :dirs_to_roodi => ["lib","config/server/datasets"] }
+
+end

Capfile

@@ -2,6 +2,7 @@ load 'deploy' if respond_to?(:namespace) # cap2 differentiator
 
 require 'rubygems'
 require 'railsless-deploy'
+require 'bundler/capistrano'
 
 load 'config/deploy'
 

Gemfile

@@ -29,6 +29,9 @@ gem 'sequel',               '~>3.0'
 gem 'mysql2'
 gem 'builder',              '~>2.1'
 
+# Misc
+gem 'awesome_print'
+
 group :development, :test do
   gem 'yard'
   gem 'shoulda',            '>=2.11'
@@ -40,8 +43,5 @@ group :development, :test do
   gem 'capistrano-ext'
   gem 'railsless-deploy'
   gem 'simplecov',                      :platforms => :ruby_19
-  gem 'rcov',                           :platforms => :ruby_18
-  gem 'metric_fu'
-  gem 'awesome_print',                  :require => 'ap'
   gem 'wirble'
 end

Gemfile.lock

@@ -1,16 +1,14 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    Saikuro (1.1.0)
     abstract (1.0.0)
-    activesupport (3.0.3)
+    activesupport (3.0.4)
     addressable (2.2.3)
-    arrayfields (4.7.4)
-    awesome_print (0.3.1)
-    biomart (0.2.0)
+    awesome_print (0.3.2)
+    biomart (0.2.1)
       builder
-    bson (1.2.0)
-    bson_ext (1.2.0)
+    bson (1.2.1)
+    bson_ext (1.2.1)
     builder (2.1.2)
     capistrano (2.5.19)
       highline
@@ -20,73 +18,38 @@ GEM
       net-ssh-gateway (>= 1.0.0)
     capistrano-ext (1.2.1)
       capistrano (>= 1.0.0)
-    capybara (0.4.0)
+    capybara (0.4.1.2)
       celerity (>= 0.7.9)
       culerity (>= 0.2.4)
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       selenium-webdriver (>= 0.0.27)
-      xpath (~> 0.1.2)
-    celerity (0.8.4)
-    childprocess (0.1.4)
+      xpath (~> 0.1.3)
+    celerity (0.8.7)
+    childprocess (0.1.6)
       ffi (~> 0.6.3)
-    chronic (0.2.3)
-      hoe (>= 1.2.1)
-    churn (0.0.13)
-      chronic (>= 0.2.3)
-      hirb
-      json_pure
-      main
-      ruby_parser (~> 2.0.4)
-      sexp_processor (~> 3.0.3)
     closure-compiler (1.0.0)
-    colored (1.2)
     crack (0.1.8)
-    culerity (0.2.12)
+    culerity (0.2.15)
     emk-sinatra-url-for (0.2.1)
       sinatra (>= 0.9.1.1)
     erubis (2.6.6)
       abstract (>= 1.0.0)
-    fattr (2.2.0)
     ffi (0.6.3)
       rake (>= 0.8.7)
-    flay (1.4.1)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    flog (2.5.0)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    haml (3.0.25)
     highline (1.6.1)
-    hirb (0.3.6)
-    hoe (2.8.0)
-      rake (>= 0.8.7)
-    hoptoad_notifier (2.4.2)
+    hoptoad_notifier (2.4.5)
       activesupport
       builder
     i18n (0.5.0)
-    json (1.4.6)
-    json_pure (1.4.6)
-    main (4.4.0)
-      arrayfields (>= 4.7.4)
-      fattr (>= 2.1.0)
+    json (1.5.1)
+    json_pure (1.5.1)
     memcache-client (1.8.5)
-    metric_fu (2.0.1)
-      Saikuro (>= 1.1.0)
-      activesupport (>= 2.0.0)
-      chronic (~> 0.2.3)
-      churn (>= 0.0.7)
-      flay (>= 1.2.1)
-      flog (>= 2.2.0)
-      rails_best_practices (>= 0.3.16)
-      rcov (>= 0.8.3.3)
-      reek (>= 1.2.6)
-      roodi (>= 2.1.0)
     mime-types (1.16)
-    mongo (1.2.0)
-      bson (>= 1.2.0)
+    mongo (1.2.1)
+      bson (>= 1.2.1)
     mongo_store (0.3.0)
       activesupport (>= 2.2)
       mongo (>= 1.0)
@@ -106,42 +69,20 @@ GEM
       rack (>= 0.9.1)
     rack-test (0.5.7)
       rack (>= 1.0)
-    rails_best_practices (0.6.5)
-      activesupport
-      colored (~> 1.2)
-      erubis (~> 2.6.6)
-      haml (~> 3.0.18)
-      i18n
-      ruby-progressbar (~> 0.0.9)
-      ruby_parser (~> 2.0.4)
     railsless-deploy (1.0.2)
     rake (0.8.7)
-    rcov (0.9.9)
-    reek (1.2.8)
-      ruby2ruby (~> 1.2)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    roodi (2.1.0)
-      ruby_parser
-    ruby-progressbar (0.0.9)
-    ruby2ruby (1.2.5)
-      ruby_parser (~> 2.0)
-      sexp_processor (~> 3.0)
-    ruby_parser (2.0.5)
-      sexp_processor (~> 3.0)
     rubytree (0.8.1)
     rubyzip (0.9.4)
-    selenium-webdriver (0.1.1)
-      childprocess (= 0.1.4)
+    selenium-webdriver (0.1.2)
+      childprocess (~> 0.1.5)
       ffi (~> 0.6.3)
       json_pure
       rubyzip
-    sequel (3.19.0)
-    sexp_processor (3.0.5)
+    sequel (3.20.0)
     shoulda (2.11.3)
-    simplecov (0.3.9)
-      simplecov-html (>= 0.3.7)
-    simplecov-html (0.3.9)
+    simplecov (0.4.0)
+      simplecov-html (~> 0.4.0)
+    simplecov-html (0.4.3)
     sinatra (1.1.2)
       rack (~> 1.1)
       tilt (~> 1.2)
@@ -150,13 +91,13 @@ GEM
       rack (>= 1.0.0)
       sinatra (>= 0.9.1)
     tilt (1.2.2)
-    vcr (1.5.0)
+    vcr (1.6.0)
     webmock (1.6.2)
       addressable (>= 2.2.2)
       crack (>= 0.1.7)
     will_paginate (2.3.15)
     wirble (0.1.3)
-    xpath (0.1.2)
+    xpath (0.1.3)
       nokogiri (~> 1.3)
     yard (0.6.4)
     yui-compressor (0.9.3)
@@ -180,7 +121,6 @@ DEPENDENCIES
   i18n
   json
   memcache-client (>= 1.8)
-  metric_fu
   mongo
   mongo_store
   mysql2
@@ -190,7 +130,6 @@ DEPENDENCIES
   rack-contrib
   rack-test (>= 0.5)
   railsless-deploy
-  rcov
   rubytree (~> 0.7)
   sequel (~> 3.0)
   shoulda (>= 2.11)

config/deploy.rb

@@ -13,6 +13,13 @@
 role :web, 'etch-dev64.internal.sanger.ac.uk'
 role :app, 'etch-dev64.internal.sanger.ac.uk'
 
+set :default_environment, {
+  'PATH'      => '/software/team87/brave_new_world/bin:/software/perl-5.8.8/bin:/usr/bin:/bin',
+  'PERL5LIB'  => '/software/team87/brave_new_world/lib/perl5:/software/team87/brave_new_world/lib/perl5/x86_64-linux-thread-multi'
+}
+
+set :bundle_cmd, '/software/team87/brave_new_world/bin/htgt-env.pl --environment Ruby19 /software/team87/brave_new_world/app/ruby-1.9.2-p0/lib/ruby/gems/1.9/bin/bundle'
+
 namespace :deploy do
   desc 'Restart Passenger'
   task :restart, :roles => :app, :except => { :no_release => true } do

config/index.json

@@ -52,6 +52,11 @@
       "phenotype":                   { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
       "sanger_expression":           { "type": "text", "indexed": true, "stored": false, "multi_valued": true },
       "expression":                  { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
+      "ma_id":                       { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
+      "ma_term":                     { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
+      "MA":                          { "type": "textTight", "indexed": true, "stored": false, "multi_valued": true },
+      "ma":                          { "type": "textTight", "indexed": true, "stored": false, "multi_valued": true },
+      "ma_ontology":                 { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
       "mp_id":                       { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
       "mp_term":                     { "type": "text", "indexed": true, "stored": true, "multi_valued": true },
       "MP":                          { "type": "textTight", "indexed": true, "stored": false, "multi_valued": true },
@@ -93,6 +98,8 @@
       { "source": "marker_symbol",              "dest": "symbol" },
       { "source": "product_status",             "dest": "product_status_str" },
       { "source": "ikmc_project_product_status","dest": "ikmc_project_product_status_str" },
+      { "source": "ma_id",                      "dest": "MA" },
+      { "source": "ma_id",                      "dest": "ma" },
       { "source": "mp_id",                      "dest": "MP" },
       { "source": "mp_id",                      "dest": "mp" },
       { "source": "go_id",                      "dest": "GO" },
@@ -132,6 +139,8 @@
       { "source": "phenotype",                  "dest": "text" },
       { "source": "sanger_expression",          "dest": "text" },
       { "source": "expression",                 "dest": "text" },
+      { "source": "ma_id",                      "dest": "text" },
+      { "source": "ma_term",                    "dest": "text" },
       { "source": "mp_id",                      "dest": "text" },
       { "source": "mp_term",                    "dest": "text" },
       { "source": "go_id",                      "dest": "text" },

config/index_builder/datasets/wtsi-phenotyping-published_images.json

@@ -0,0 +1,20 @@
+{
+  "datasource": "wtsi-phenotyping",
+  "indexing": {
+    "allow_document_creation": false,
+    "days_between_downlads": 0,
+    "attribute_map": [
+      { "attr": "published_images_colony_prefix", "idx": "colony_prefix", "use_to_map": true },
+      { "attr": "published_images_annotations",   "idx": "ignore_me" }
+    ],
+    "filters": {},
+    "concatenated_ontology_terms": {
+      "attr":     "published_images_annotations",
+      "split_on": "; ",
+      "ontologies": {
+        "^MA\\:\\d+$": { "term": "ma_id", "term_name": "ma_term", "breadcrumb": "ma_ontology" },
+        "^MP\\:\\d+$": { "term": "mp_id", "term_name": "mp_term", "breadcrumb": "mp_ontology" }
+      }
+    }
+  }
+}

config/index_builder/index_builder.json

@@ -7,6 +7,7 @@
     "ikmc-kermits",
     "wtsi-phenotyping-heatmap",
     "wtsi-phenotyping-adult_expression",
+    "wtsi-phenotyping-published_images",
     "eurexpress",
     "europhenome",
     "emma-strains",

config/server/datasets/emma-strains/custom_sort.rb

@@ -5,20 +5,13 @@ def emma_strains_sort_results( results )
       sorted_results = {}
 
       results.each do |result|
-        joined_attribute = @config[:searching][:joined_attribute].to_sym
+        joined_attribute = result[ @config[:searching][:joined_attribute].to_sym ]
+        emma_id          = result[:emma_id]
 
-        unless sorted_results[ result[ joined_attribute ] ]
-          sorted_results[ result[ joined_attribute ] ] = {}
-        end
-
-        unless sorted_results[ result[ joined_attribute ] ][ result[:emma_id] ]
-          sorted_results[ result[ joined_attribute ] ][ result[:emma_id] ] = {
-            :references   => {},
-            :availability => []
-          }
-        end
+        sorted_results[joined_attribute]          ||= {}
+        sorted_results[joined_attribute][emma_id] ||= { :references => {}, :availability => [] }
 
-        emma_record = sorted_results[ result[ joined_attribute ] ][ result[:emma_id] ]
+        emma_record = sorted_results[joined_attribute][emma_id]
 
         # Add singular info first...
         singles = [
@@ -42,17 +35,12 @@ def emma_strains_sort_results( results )
         emma_record[:allele_name] = fix_superscript_text_in_attribute(result[:alls_form])
 
         # References...
-        unless result[:pubmed_id].nil?
-          emma_record[:references][result[:pubmed_id]] = {
-            :pubmed_id => result[:pubmed_id],
-            :reference => result[:reference]
-          }
-        end
+        pubmed_id = result[:pubmed_id]
+        emma_record[:references][pubmed_id] = { :pubmed_id => pubmed_id, :reference => result[:reference] } unless pubmed_id.nil?
 
         # Availability...
-        unless result[:availability].nil?
-          emma_record[:availability].push(result[:availability])
-        end
+        availability = result[:availability]
+        emma_record[:availability].push(availability) unless availability.nil?
       end
 
       return sorted_results

config/server/datasets/ensembl-mouse-homologs/custom_sort.rb

@@ -5,29 +5,21 @@ def ensembl_mouse_homologs_sort_results( results )
       sorted_results = {}
 
       results.each do |result|
-        joined_attribute = @config[:searching][:joined_attribute].to_sym
+        joined_attribute = result[ @config[:searching][:joined_attribute].to_sym ]
 
-        if sorted_results[ result[ joined_attribute ] ].nil?
-          sorted_results[ result[ joined_attribute ] ] = {}
-
-          result.keys.each do |key|
-            sorted_results[ result[ joined_attribute ] ][key] = []
-          end
-        end
-
-        result_data = sorted_results[ result[ joined_attribute ] ]
+        sorted_results[joined_attribute] ||= {}
+        result_data                        = sorted_results[joined_attribute]
 
         result.keys.each do |key|
+          result_data[key] ||= []
           result_data[key].push( result[key] )
         end
       end
 
       # Finally, ensure that the data in the arrays is unique
       sorted_results.each do |key,result_data|
-        result_data.keys.each do |field|
-          if result_data[field].is_a?(Array)
-            result_data[field].uniq!
-          end
+        result_data.keys.each do |field,field_data|
+          field_data.uniq! if field_data.is_a?(Array)
         end
       end