* Update our dependencies to require Builder 3.0+

* Refactor the test suite into seperate modules
* Use VCR to mock/stub the http requests

.gitignore

@@ -4,3 +4,4 @@ pkg/*
 simplecov*
 tmp/*.json
 tmp/metric_fu*
+test/vcr_cassettes*

Gemfile.lock

@@ -1,22 +1,32 @@
 PATH
   remote: .
   specs:
-    biomart (0.2.1)
-      builder
+    biomart (0.2.2)
+      builder (>= 3.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    builder (2.1.2)
+    addressable (2.2.4)
+    builder (3.0.0)
+    crack (0.1.8)
+    rake (0.8.7)
     shoulda (2.11.3)
     simplecov (0.4.0)
       simplecov-html (~> 0.4.0)
     simplecov-html (0.4.3)
+    vcr (1.7.2)
+    webmock (1.6.2)
+      addressable (>= 2.2.2)
+      crack (>= 0.1.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   biomart!
+  rake
   shoulda (>= 2.10)
   simplecov
+  vcr
+  webmock

biomart.gemspec

@@ -19,7 +19,10 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency("builder", [">= 0"])
+  s.add_runtime_dependency("builder", [">= 3.0"])
+  s.add_development_dependency("rake", [">= 0"])
   s.add_development_dependency("shoulda", [">= 2.10"])
   s.add_development_dependency("simplecov", [">= 0"])
+  s.add_development_dependency("vcr", [">= 0"])
+  s.add_development_dependency("webmock", [">= 0"])
 end

test/test_biomart_attribute.rb

@@ -0,0 +1,31 @@
+require 'test_helper'
+
+class BiomartAttributeTest < Test::Unit::TestCase
+  def setup
+    VCR.insert_cassette('test_biomart_attribute')
+    @htgt     = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+    @htgt_alt = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+  end
+
+  def teardown
+    VCR.eject_cassette
+  end
+
+  context "A Biomart::Attribute instance" do
+    setup do
+      @kermits = @htgt.datasets["kermits"]
+    end
+
+    should "have basic metadata" do
+      true_false  = [true,false]
+      ens_gene_id = @kermits.attributes["ensembl_gene_id"]
+
+      assert( !ens_gene_id.name.nil?, "Biomart::Attribute.name is nil." )
+      assert( !ens_gene_id.display_name.nil?, "Biomart::Attribute.display_name is nil." )
+
+      assert( true_false.include?( ens_gene_id.hidden? ), "Biomart::Attribute.hidden? is not returning true/false." )
+      assert( true_false.include?( ens_gene_id.default? ), "Biomart::Attribute.default? is not returning true/false." )
+    end
+  end
+
+end

test/test_biomart_database.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class BiomartDatabaseTest < Test::Unit::TestCase
+  def setup
+    VCR.insert_cassette('test_biomart_database')
+    @htgt     = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+    @htgt_alt = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+  end
+
+  def teardown
+    VCR.eject_cassette
+  end
+
+  context "A Biomart::Database instance" do
+    setup do
+      @htgt_database = @htgt.databases["htgt"]
+    end
+
+    should "have basic metadata" do
+      true_false  = [true,false]
+      assert( @htgt_database.display_name, "Biomart::Database does not have a 'display name'." )
+      assert( @htgt_database.name, "Biomart::Database does not have a 'name'." )
+      assert( @htgt_database.visible != nil, "Biomart::Database does not have a 'visible' flag." )
+      assert( true_false.include?( @htgt_database.visible? ), "Biomart::Database.visible? is not returning true/false." )
+    end
+
+    should "have datasets" do
+      assert( @htgt_database.list_datasets.is_a?( Array ), "Biomart::Database.list_datasets is not returning an array." )
+      assert( @htgt_database.list_datasets.include?( "htgt_targ" ), "Biomart::Database dataset parsing is off - htgt_targ is not in htgt!" )
+      assert( @htgt_database.datasets["htgt_targ"].is_a?( Biomart::Dataset ), "Biomart::Database is not creating Biomart::Dataset objects." )
+    end
+  end
+
+end

test/test_biomart_dataset.rb

@@ -0,0 +1,272 @@
+require 'test_helper'
+
+class BiomartDatasetTest < Test::Unit::TestCase
+  def setup
+    VCR.insert_cassette('test_biomart_dataset')
+    @htgt     = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+    @htgt_alt = Biomart::Server.new('http://www.sanger.ac.uk/htgt/biomart')
+  end
+
+  def teardown
+    VCR.eject_cassette
+  end
+
+  context "A Biomart::Dataset instance" do
+    setup do
+      @htgt_targ   = @htgt.datasets["htgt_targ"]
+      @htgt_trap   = @htgt.datasets["htgt_trap"]
+      @kermits     = @htgt.datasets["kermits"]
+      @ensembl     = @htgt.datasets["mmusculus_gene_ensembl"]
+      @ensembl_var = Biomart::Dataset.new( "http://www.ensembl.org/biomart", { :name => "hsapiens_snp" } )
+      @emma        = Biomart::Dataset.new( "http://www.emmanet.org/biomart", { :name => "strains" } )
+      @dcc         = Biomart::Dataset.new( "http://www.knockoutmouse.org/biomart", { :name => "dcc" } )
+      @mgi         = Biomart::Dataset.new( "http://biomart.informatics.jax.org/biomart", { :name => "markers" } )
+    end
+
+    should "have basic metadata" do
+      assert( @htgt_targ.display_name, "Biomart::Dataset does not have a 'display name'." )
+      assert( @htgt_targ.name, "Biomart::Dataset does not have a 'name'." )
+      assert( @htgt_targ.visible != nil, "Biomart::Dataset does not have a 'visible' flag." )
+    end
+
+    should "have filters" do
+      assert( @htgt_targ.list_filters.is_a?( Array ), "Biomart::Dataset.list_filters is not returning an array." )
+      assert( @htgt_targ.list_filters.include?( "ensembl_gene_id" ), "Biomart::Dataset filter parsing is off - ensembl_gene_id is not in htgt_targ!" )
+      assert( @mgi.list_filters.include?( "ancestor_term_1023_filter" ), "Biomart::Dataset filter parsing is off - ancestor_term_1023_filter is not in mgi markers!" )
+      assert( @kermits.filters["ensembl_gene_id"].is_a?( Biomart::Filter ), "Biomart::Dataset is not creating Biomart::Filter objects." )
+    end
+
+    should "have attributes" do
+      assert( @htgt_targ.list_attributes.is_a?( Array ), "Biomart::Dataset.list_attributes is not returning an array." )
+      assert( @htgt_targ.list_attributes.include?( "ensembl_gene_id" ), "Biomart::Dataset attribute parsing is off - ensembl_gene_id is not in htgt_targ!" )
+      assert( @kermits.attributes["ensembl_gene_id"].is_a?( Biomart::Attribute ), "Biomart::Dataset is not creating Biomart::Attribute objects." )
+    end
+
+    should "perform count queries" do
+      htgt_count = @htgt_targ.count()
+      assert( htgt_count.is_a?( Integer ), "Biomart::Dataset.count is not returning integers." )
+      assert( htgt_count > 0, "Biomart::Dataset.count is returning zero - this is wrong!" )
+
+      htgt_count_single_filter = @htgt_targ.count( :filters => { "is_eucomm" => "1" } )
+      assert( htgt_count_single_filter.is_a?( Integer ), "Biomart::Dataset.count (with single filter) is not returning integers." )
+      assert( htgt_count_single_filter > 0, "Biomart::Dataset.count (with single filter) is returning zero - this is wrong!" )
+
+      htgt_count_single_filter_group_value = @htgt_targ.count( :filters => { "marker_symbol" => ["Cbx1","Cbx7","Art4"] } )
+      assert( htgt_count_single_filter_group_value.is_a?( Integer ), "Biomart::Dataset.count (with single filter, group value) is not returning integers." )
+      assert( htgt_count_single_filter_group_value > 0, "Biomart::Dataset.count (with single filter, group value) is returning zero - this is wrong!" )
+    end
+
+    should "perform search queries" do
+      search = @htgt_trap.search()
+      assert( search.is_a?( Hash ), "Biomart::Dataset.search (no options) is not returning a hash." )
+      assert( search[:data].is_a?( Array ), "Biomart::Dataset.search[:data] (no options) is not returning an array." )
+
+      search1 = @htgt_targ.search( :filters => { "marker_symbol" => "Cbx1" }, :process_results => true )
+      assert( search1.is_a?( Array ), "Biomart::Dataset.search (filters defined with processing) is not returning an array." )
+      assert( search1.first.is_a?( Hash ), "Biomart::Dataset.search (filters defined with processing) is not returning an array of hashes." )
+      assert( search1.first["marker_symbol"] == "Cbx1", "Biomart::Dataset.search (filters defined with processing) is not returning the correct info." )
+
+      search2 = @htgt_targ.search( :filters => { "marker_symbol" => "Cbx1" }, :attributes => ["marker_symbol","ensembl_gene_id"], :process_results => true )
+      assert( search2.is_a?( Array ), "Biomart::Dataset.search (filters and attributes defined with processing) is not returning an array." )
+      assert( search2.first.is_a?( Hash ), "Biomart::Dataset.search (filters and attributes defined with processing) is not returning an array of hashes." )
+      assert( search2.first["marker_symbol"] == "Cbx1", "Biomart::Dataset.search (filters and attributes defined with processing) is not returning the correct info." )
+    end
+
+    should "perform search queries whilst altering the timeout property" do
+      search = @htgt_trap.search( :timeout => 60 )
+      assert( search.is_a?( Hash ), "Biomart::Dataset.search (no options except per-request timeout) is not returning a hash." )
+      assert( search[:data].is_a?( Array ), "Biomart::Dataset.search[:data] (no options except per-request timeout) is not returning an array." )
+
+      Biomart.timeout = 60
+      search = @htgt_trap.search()
+      assert( search.is_a?( Hash ), "Biomart::Dataset.search (no options except global timeout) is not returning a hash." )
+      assert( search[:data].is_a?( Array ), "Biomart::Dataset.search[:data] (no options except global timeout) is not returning an array." )
+    end
+
+    should "handle search queries that will generate poorly formatted TSV data" do
+      search = @htgt_targ.search(
+        :filters => { "mgi_accession_id" => [ "MGI:1921569", "MGI:1913402", "MGI:1913300" ] },
+        :attributes => [
+          "is_eucomm", "is_komp_csd", "is_komp_regeneron", "is_norcomm",
+          "is_mgp", "mgi_accession_id", "marker_symbol", "ensembl_gene_id",
+          "status", "status_code", "status_type", "status_description",
+          "status_sequence", "pipeline_stage", "ikmc_project_id", "bac",
+          "design_id", "design_plate", "design_well", "intvec_plate",
+          "intvec_well", "intvec_distribute", "targvec_plate", "targvec_well",
+          "targvec_distribute", "backbone", "cassette", "allele_name",
+          "escell_clone", "escell_distribute", "escell_line", "colonies_picked",
+          "is_latest_for_gene", "targeted_trap"
+        ]
+      )
+      assert( search.is_a?( Hash ), "Biomart::Dataset.search (no options) is not returning a hash. (HTGT Query)" )
+      assert( search[:data].is_a?( Array ), "Biomart::Dataset.search[:data] (no options) is not returning an array. (HTGT Query)" )
+      assert( search[:data].size > 20, "Biomart::Dataset.search[:data] for poorly formatted TSV data is empty. (HTGT Query)" )
+
+      search2 = @emma.search(
+        :filters    => { "emma_id" => ["EM:03629"] },
+        :attributes => [
+          "emma_id", "international_strain_name", "synonym", "maintained_background", 
+          "mutation_main_type", "mutation_sub_type", "alls_form", "genetic_description", 
+          "phenotype_description", "reference", "pubmed_id", "availability", "owner"
+        ]
+      )
+
+      assert( search2.is_a?( Hash ), "Biomart::Dataset.search (no options) is not returning a hash. (EMMA Query)" )
+      assert( search2[:data].is_a?( Array ), "Biomart::Dataset.search[:data] (no options) is not returning an array. (EMMA Query)" )
+      assert( search2[:data].size > 0, "Biomart::Dataset.search[:data] for poorly formatted TSV data is empty. (EMMA Query)" )
+    end
+
+    should "perform federated search queries" do
+      search_opts = {
+        :filters => {
+          "status" => [
+            "Mice - Genotype confirmed", "Mice - Germline transmission",
+            "Mice - Microinjection in progress", "ES Cells - Targeting Confirmed"
+          ]
+        },
+        :attributes => [ "marker_symbol", "mgi_accession_id", "status" ],
+        :federate => [
+          {
+            :dataset => @ensembl,
+            :filters => { "chromosome_name" => "1", "start" => "1", "end" => "10000000" },
+            :attributes => []
+          }
+        ]
+      }
+
+      results = @htgt_targ.search( search_opts )
+
+      assert( results.is_a?(Hash), "Biomart::Dataset.search is not returning a hash. [federated search]" )
+      assert( results[:data].is_a?(Array), "Biomart::Dataset.search[:data] is not returning an array. [federated search]" )
+      assert( results[:data][0].size === 3, "Biomart::Dataset.search[:data] is not returning 3 attributes. [federated search]" )
+      assert( results[:headers].size === 3, "Biomart::Dataset.search[:headers] is not returning 3 elements. [federated search]" )
+
+      assert_raise( Biomart::ArgumentError ) { @htgt_targ.count( search_opts ) }
+
+      assert_raise Biomart::ArgumentError do
+        search_opts[:federate] = [
+          {
+            :dataset => "mmusculus_gene_ensembl",
+            :filters => { "chromosome_name" => "1", "start" => "1", "end" => "10000000" },
+            :attributes => []
+          }
+        ]
+        results = @htgt_targ.search( search_opts )
+      end
+
+      assert_raise Biomart::ArgumentError do
+        search_opts[:federate] = {
+          :dataset => "mmusculus_gene_ensembl",
+          :filters => { "chromosome_name" => "1", "start" => "1", "end" => "10000000" },
+          :attributes => []
+        }
+        results = @htgt_targ.search( search_opts )
+      end
+
+      assert_raise Biomart::ArgumentError do
+        search_opts[:federate] = [
+          {
+            :dataset => @ensembl,
+            :filters => { "chromosome_name" => "1", "start" => "1", "end" => "10000000" },
+            :attributes => []
+          },
+          {
+            :dataset => @ensembl,
+            :filters => { "chromosome_name" => "1", "start" => "1", "end" => "10000000" },
+            :attributes => []
+          }
+        ]
+        results = @htgt_targ.search( search_opts )
+      end
+    end
+
+    should "perform search queries with the :required_attributes option" do
+      search_opts = {
+        :filters => {
+          "chromosome_name" => "1",
+          "start"           => "1",
+          "end"             => "10000000"
+        },
+        :attributes => [
+          "ensembl_gene_id", "ensembl_transcript_id",
+          "mouse_paralog_ensembl_gene", "mouse_paralog_chromosome"
+        ],
+        :required_attributes => ["mouse_paralog_ensembl_gene"]
+      }
+
+      results = @ensembl.search( search_opts )
+
+      assert( results.is_a?(Hash), "Biomart::Dataset.search is not returning a hash. [required_attributes search]" )
+      assert( results[:data].is_a?(Array), "Biomart::Dataset.search[:data] is not returning an array. [required_attributes search]" )
+      results[:data].each do |data_row|
+        assert_equal( false, data_row[2].nil?, "The required_attributes search has not filtered out nil values." )
+      end
+
+      assert_raise( Biomart::ArgumentError ) { @ensembl.count( search_opts ) }
+      assert_raise Biomart::ArgumentError do
+        search_opts[:required_attributes] = "mouse_paralog_ensembl_gene"
+        @ensembl.search( search_opts )
+      end
+
+      results = @dcc.search(
+        :filters => {
+          "marker_symbol" => [
+            "Lrrc32", "Dub3", "Hs3st4", "Hs3st4", "Hs3st4", "Hs3st4",
+            "Hs3st4", "Hs3st4", "Hs3st4", "Tcrg-C", "Gm5195", "Gm5198",
+            "Gm5199", "Gm5625", "Rpl13-ps2", "Gm5664", "Gm5928", "Gm6035",
+            "Gm6049"
+          ]
+        },
+        :required_attributes => ["ikmc_project","ikmc_project_id"],
+        :process_results => true
+      )
+
+      results.each do |data_row|
+        assert_equal( false, data_row["ikmc_project"].nil?, "The required_attributes search has not filtered out nil values." )
+        assert_equal( false, data_row["ikmc_project_id"].nil?, "The required_attributes search has not filtered out nil values." )
+      end
+    end
+
+    should "perform searches that involve boolean filters" do
+      search_opts = {
+        :filters         => { 'with_variation_annotation' => true, 'ensembl_gene' => 'ENSG00000244734' },
+        :attributes      => [ 'refsnp_id','chr_name','chrom_start' ],
+        :process_results => true
+      }
+
+      true_results = {}
+      assert_nothing_raised( Biomart::BiomartError ) { true_results = @ensembl_var.search( search_opts ) }
+      assert( !true_results.empty?, "The search using a boolean filter is empty." )
+
+      search_opts[:filters].merge!({ 'with_variation_annotation' => 'included' })
+      true_results2 = {}
+      assert_nothing_raised( Biomart::BiomartError ) { true_results2 = @ensembl_var.search( search_opts ) }
+      assert( !true_results2.empty?, "The search using a boolean filter is empty." )
+      assert_equal( true_results, true_results2, "Using 'included' for a boolean filter does not give the same result as 'true'." )
+
+      search_opts[:filters].merge!({ 'with_variation_annotation' => 'only' })
+      true_results3 = {}
+      assert_nothing_raised( Biomart::BiomartError ) { true_results3 = @ensembl_var.search( search_opts ) }
+      assert( !true_results3.empty?, "The search using a boolean filter is empty." )
+      assert_equal( true_results, true_results3, "Using 'only' for a boolean filter does not give the same result as 'true'." )
+
+      search_opts[:filters].merge!({ 'with_variation_annotation' => false })
+      false_results = {}
+      assert_nothing_raised( Biomart::BiomartError ) { false_results = @ensembl_var.search( search_opts ) }
+      assert( !false_results.empty?, "The search using a boolean filter is empty." )
+
+      search_opts[:filters].merge!({ 'with_variation_annotation' => 'excluded' })
+      false_results2 = {}
+      assert_nothing_raised( Biomart::BiomartError ) { false_results2 = @ensembl_var.search( search_opts ) }
+      assert( !false_results2.empty?, "The search using a boolean filter is empty." )
+      assert_equal( false_results, false_results2, "Using 'excluded' for a boolean filter does not give the same result as 'false'." )
+
+      search_opts[:filters].merge!({ 'with_variation_annotation' => 'flibble' })
+      assert_raise( Biomart::ArgumentError ) { @ensembl_var.search( search_opts ) }
+
+      search_opts[:filters].merge!({ 'with_variation_annot' => true })
+      assert_raise( Biomart::ArgumentError ) { @ensembl_var.search( search_opts ) }
+    end
+  end
+
+end