From af7292489c08830991510ef616e39889d32d0c17 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Thu, 11 Jun 2015 16:49:57 +0200 Subject: [PATCH 1/6] Retries calls if they fail (updates #10) --- lib/commands/unipept/api_runner.rb | 18 ++++++++++++++++-- lib/retryable_typhoeus.rb | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 lib/retryable_typhoeus.rb diff --git a/lib/commands/unipept/api_runner.rb b/lib/commands/unipept/api_runner.rb index d08228b5..aac7f469 100644 --- a/lib/commands/unipept/api_runner.rb +++ b/lib/commands/unipept/api_runner.rb @@ -1,3 +1,5 @@ +require_relative '../../retryable_typhoeus' + module Unipept class Commands::ApiRunner < Cri::CommandRunner attr_reader :configuration @@ -114,7 +116,7 @@ def run batch_iterator = Unipept::BatchIterator.new(batch_size) batch_iterator.iterate(input_iterator) do |input_slice, batch_id, fasta_mapper| - request = Typhoeus::Request.new( + request = ::RetryableTyphoeus::Request.new( @url, method: :post, body: construct_request_body(input_slice), @@ -122,7 +124,19 @@ def run headers: { 'User-Agent' => @user_agent } ) - request.on_complete do |resp| + # Failure callback: retry if retries not exceeded + request.on_failure do |resp| + if resp.request.retries <= 0 + block = handle_response(resp, batch_id, fasta_mapper) + batch_order.wait(batch_id, &block) + else + resp.request.retries -= 1 + hydra.queue_front resp.request + end + end + + # Success callback + request.on_success do |resp| block = handle_response(resp, batch_id, fasta_mapper) batch_order.wait(batch_id, &block) end diff --git a/lib/retryable_typhoeus.rb b/lib/retryable_typhoeus.rb new file mode 100644 index 00000000..bca3f952 --- /dev/null +++ b/lib/retryable_typhoeus.rb @@ -0,0 +1,28 @@ +# Retryable Typheous +# Inspiration: https://gist.github.com/kunalmodi/2939288 +# Patches the request and hydra to allow requests to get resend when they fail + +module RetryableTyphoeus + require 'typhoeus' + + include Typhoeus + + DEFAULT_RETRIES = 10 + + class Request < Typhoeus::Request + def initialize(base_url, options = {}) + @retries = (options.delete(:retries) || DEFAULT_RETRIES) + + super + end + + def retries=(retries) + @retries = retries + end + + def retries + @retries ||= 0 + end + end + +end From e32375f5a4c180836802ae39f032dc6f4a386419 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Thu, 11 Jun 2015 17:35:02 +0200 Subject: [PATCH 2/6] Use attr_accessor instead of methods --- lib/retryable_typhoeus.rb | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/retryable_typhoeus.rb b/lib/retryable_typhoeus.rb index bca3f952..d116bb3e 100644 --- a/lib/retryable_typhoeus.rb +++ b/lib/retryable_typhoeus.rb @@ -10,19 +10,12 @@ module RetryableTyphoeus DEFAULT_RETRIES = 10 class Request < Typhoeus::Request + attr_accessor :retries + def initialize(base_url, options = {}) @retries = (options.delete(:retries) || DEFAULT_RETRIES) super end - - def retries=(retries) - @retries = retries - end - - def retries - @retries ||= 0 - end end - end From a603b5d6bbf399d2424001c4fb8430a22acd54e3 Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Fri, 12 Jun 2015 15:10:30 +0200 Subject: [PATCH 3/6] don't use get in accessor names --- .rubocop.yml | 2 -- lib/commands/unipept/api_runner.rb | 4 ++-- test/commands/unipept/test_api_runner.rb | 12 ++++++------ test/test_formatters.rb | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index e3590e55..ff8d785e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,8 +10,6 @@ Style/ClassAndModuleChildren: Enabled: false # disable for now -Style/AccessorMethodName: - Enabled: false Style/ClassVars: Enabled: false Style/Documentation: diff --git a/lib/commands/unipept/api_runner.rb b/lib/commands/unipept/api_runner.rb index 9ce74233..27c2662d 100644 --- a/lib/commands/unipept/api_runner.rb +++ b/lib/commands/unipept/api_runner.rb @@ -21,13 +21,13 @@ def initialize(args, opts, cmd) # - the host # - the user agent def set_configuration - @host = get_host + @host = host @user_agent = 'Unipept CLI - unipept ' + Unipept::VERSION end # Returns the host. If a value is defined by both an option and the config # file, the value of the option is used. - def get_host + def host # find host in opts first host = options[:host] ? options[:host] : @configuration['host'] diff --git a/test/commands/unipept/test_api_runner.rb b/test/commands/unipept/test_api_runner.rb index 071485a9..7a185f3c 100644 --- a/test/commands/unipept/test_api_runner.rb +++ b/test/commands/unipept/test_api_runner.rb @@ -22,14 +22,14 @@ def test_config_host runner = new_runner('test', { host: 'http://param_host' }, %w(a b c)) runner.options.delete(:host) runner.configuration['host'] = 'http://config_host' - host = runner.get_host + host = runner.host assert_equal('http://config_host', host) end def test_param_host runner = new_runner('test', { host: 'http://param_host' }, %w(a b c)) runner.configuration.delete('host') - host = runner.get_host + host = runner.host assert_equal('http://param_host', host) end @@ -39,7 +39,7 @@ def test_no_host runner.options.delete(:host) _out, err = capture_io_while do assert_raises SystemExit do - runner.get_host + runner.host end end assert(err.start_with? 'WARNING: no host has been set') @@ -48,19 +48,19 @@ def test_no_host def test_host_priority runner = new_runner('test', { host: 'http://param_host' }, %w(a b c)) runner.configuration['host'] = 'http://config_host' - host = runner.get_host + host = runner.host assert_equal('http://param_host', host) end def test_http_host runner = new_runner('test', { host: 'param_host' }, %w(a b c)) - host = runner.get_host + host = runner.host assert_equal('http://param_host', host) end def test_https_host runner = new_runner('test', { host: 'https://param_host' }, %w(a b c)) - host = runner.get_host + host = runner.host assert_equal('https://param_host', host) end diff --git a/test/test_formatters.rb b/test/test_formatters.rb index 985d9747..36e4efd3 100644 --- a/test/test_formatters.rb +++ b/test/test_formatters.rb @@ -38,7 +38,7 @@ def formatter end def test_header - assert_equal('', formatter.header(TestObject.get_object)) + assert_equal('', formatter.header(TestObject.test_object)) end def test_type @@ -46,7 +46,7 @@ def test_type end def test_format - assert_equal(TestObject.get_object, formatter.format(TestObject.get_object)) + assert_equal(TestObject.test_object, formatter.format(TestObject.test_object)) end end @@ -56,7 +56,7 @@ def formatter end def test_header - assert_equal('', formatter.header(TestObject.get_object)) + assert_equal('', formatter.header(TestObject.test_object)) end def test_type @@ -64,7 +64,7 @@ def test_type end def test_format - assert_equal(TestObject.as_json, formatter.format(TestObject.get_object)) + assert_equal(TestObject.as_json, formatter.format(TestObject.test_object)) end end @@ -75,7 +75,7 @@ def formatter def test_header fasta = [['peptide', '>test']] - object = [TestObject.get_object, TestObject.get_object] + object = [TestObject.test_object, TestObject.test_object] assert_equal(TestObject.as_csv_header, formatter.header(object)) assert_equal('fasta_header,' + TestObject.as_csv_header, formatter.header(object, fasta)) end @@ -85,14 +85,14 @@ def test_type end def test_format - object = [TestObject.get_object, TestObject.get_object] + object = [TestObject.test_object, TestObject.test_object] csv = [TestObject.as_csv, TestObject.as_csv, ''].join("\n") assert_equal(csv, formatter.format(object)) end def test_format_with_fasta fasta = [['>test', '5']] - object = [TestObject.get_object, TestObject.get_object] + object = [TestObject.test_object, TestObject.test_object] csv = ['>test,' + TestObject.as_csv, '>test,' + TestObject.as_csv, ''].join("\n") assert_equal(csv, formatter.format(object, fasta)) end @@ -104,7 +104,7 @@ def formatter end def test_header - assert_equal('', formatter.header(TestObject.get_object)) + assert_equal('', formatter.header(TestObject.test_object)) end def test_type @@ -112,12 +112,12 @@ def test_type end def test_format - assert_equal(TestObject.as_xml, formatter.format(TestObject.get_object)) + assert_equal(TestObject.as_xml, formatter.format(TestObject.test_object)) end end class TestObject - def self.get_object + def self.test_object JSON.parse('{"integer": 5, "string": "string", "list": ["a", 2, false]}') end From a56f133f176bd6af90d912931a7e3c89dcb58a7c Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Fri, 12 Jun 2015 15:18:37 +0200 Subject: [PATCH 4/6] use modifier if for single line body --- .rubocop.yml | 2 -- lib/commands/peptfilter.rb | 4 +--- lib/commands/unipept/config.rb | 4 +--- lib/commands/uniprot.rb | 4 +--- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index ff8d785e..8dacae2e 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,8 +14,6 @@ Style/ClassVars: Enabled: false Style/Documentation: Enabled: false -Style/IfUnlessModifier: - Enabled: false Style/RescueModifier: Enabled: false Metrics/AbcSize: diff --git a/lib/commands/peptfilter.rb b/lib/commands/peptfilter.rb index dfd4cd77..b638079c 100644 --- a/lib/commands/peptfilter.rb +++ b/lib/commands/peptfilter.rb @@ -33,9 +33,7 @@ class Peptfilter end pept = pept.chomp - if Peptfilter.filter(pept, minlen, maxlen, lacks, contains) - puts pept - end + puts pept if Peptfilter.filter(pept, minlen, maxlen, lacks, contains) end end end diff --git a/lib/commands/unipept/config.rb b/lib/commands/unipept/config.rb index e547f0a6..51a1989c 100644 --- a/lib/commands/unipept/config.rb +++ b/lib/commands/unipept/config.rb @@ -1,9 +1,7 @@ module Unipept class Commands::Config < Cri::CommandRunner def run - if arguments.size == 0 || arguments.size > 2 - abort command.help - end + abort command.help if arguments.size == 0 || arguments.size > 2 key, value = *arguments diff --git a/lib/commands/uniprot.rb b/lib/commands/uniprot.rb index 9d354ed4..cae8c18b 100644 --- a/lib/commands/uniprot.rb +++ b/lib/commands/uniprot.rb @@ -62,9 +62,7 @@ def self.get_uniprot_entry(accession, format) else # other format has been specified, just download and output resp = Typhoeus.get("http://www.uniprot.org/uniprot/#{accession}.#{format}") - if resp.success? - resp.response_body - end + resp.response_body if resp.success? end end end From a31d5706f8509cd465398da9d724425ccf4f5e1e Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Fri, 12 Jun 2015 15:27:28 +0200 Subject: [PATCH 5/6] reenable extra rubocop checks --- .rubocop.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 8dacae2e..f91664ec 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -20,11 +20,7 @@ Metrics/AbcSize: Enabled: false Metrics/ClassLength: Enabled: false -Metrics/CyclomaticComplexity: - Enabled: false Metrics/LineLength: Enabled: false Metrics/MethodLength: Enabled: false -Metrics/PerceivedComplexity: - Enabled: false From eb6dd08beb1e59e28ab911a1950ddb83fa73ad1e Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Fri, 12 Jun 2015 15:54:05 +0200 Subject: [PATCH 6/6] bump version --- README.md | 6 ++--- VERSION | 2 +- unipept.gemspec | 61 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 656a33b3..9e59ed17 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ The Unipept CLI is available as a *gem*. This means it can easily be installed w ```bash $ gem install unipept -Successfully installed unipept-0.7.1 -Parsing documentation for unipept-0.7.1 +Successfully installed unipept-0.8.0 +Parsing documentation for unipept-0.8.0 Done installing documentation for unipept after 0 seconds 1 gem installed ``` @@ -33,7 +33,7 @@ After successful installation, the unipept command should be available: ```bash $ unipept -v -0.7.1 +0.8.0 ``` The help can be accessed by running `unipept -h`. diff --git a/VERSION b/VERSION index 39e898a4..a3df0a69 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +0.8.0 diff --git a/unipept.gemspec b/unipept.gemspec index 1294a6f0..ae74815c 100644 --- a/unipept.gemspec +++ b/unipept.gemspec @@ -2,16 +2,15 @@ # DO NOT EDIT THIS FILE DIRECTLY # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' # -*- encoding: utf-8 -*- -# stub: unipept 0.7.1 ruby lib +# stub: unipept 0.8.0 ruby lib Gem::Specification.new do |s| s.name = "unipept" - s.version = "0.7.1" + s.version = "0.8.0" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= - s.require_paths = ["lib"] s.authors = ["Toon Willems", "Bart Mesuere", "Tom Naessens"] - s.date = "2015-05-20" + s.date = "2015-06-12" s.description = " Command line interface to the Unipept (http://unipept.ugent.be) web services\n (pept2lca, taxa2lca, pept2taxa and taxonomy) and some utility commands for\n handling proteins using the command line.\n" s.email = "unipept@ugent.be" s.executables = ["unipept", "prot2pept", "peptfilter", "uniprot"] @@ -21,6 +20,8 @@ Gem::Specification.new do |s| ] s.files = [ ".document", + ".rubocop.yml", + ".travis.yml", "Gemfile", "Gemfile.lock", "LICENSE.txt", @@ -31,26 +32,47 @@ Gem::Specification.new do |s| "bin/prot2pept", "bin/unipept", "bin/uniprot", - "lib/unipept.rb", - "lib/unipept/batch_order.rb", - "lib/unipept/commands.rb", - "lib/unipept/commands/api_runner.rb", - "lib/unipept/commands/pept2lca.rb", - "lib/unipept/commands/pept2prot.rb", - "lib/unipept/commands/pept2taxa.rb", - "lib/unipept/commands/taxa2lca.rb", - "lib/unipept/commands/taxonomy.rb", - "lib/unipept/configuration.rb", - "lib/unipept/formatters.rb", - "lib/unipept/version.rb", + "lib/batch_iterator.rb", + "lib/batch_order.rb", + "lib/commands.rb", + "lib/commands/peptfilter.rb", + "lib/commands/prot2pept.rb", + "lib/commands/unipept.rb", + "lib/commands/unipept/api_runner.rb", + "lib/commands/unipept/config.rb", + "lib/commands/unipept/pept2lca.rb", + "lib/commands/unipept/pept2prot.rb", + "lib/commands/unipept/pept2taxa.rb", + "lib/commands/unipept/taxa2lca.rb", + "lib/commands/unipept/taxonomy.rb", + "lib/commands/uniprot.rb", + "lib/configuration.rb", + "lib/formatters.rb", + "lib/version.rb", + "test/commands/test_peptfilter.rb", + "test/commands/test_prot2pept.rb", + "test/commands/test_unipept.rb", + "test/commands/test_uniprot.rb", + "test/commands/unipept/test_api_runner.rb", + "test/commands/unipept/test_config.rb", + "test/commands/unipept/test_pept2lca.rb", + "test/commands/unipept/test_pept2prot.rb", + "test/commands/unipept/test_pept2taxa.rb", + "test/commands/unipept/test_taxa2lca.rb", + "test/commands/unipept/test_taxonomy.rb", "test/helper.rb", - "test/test_unipept.rb", + "test/test_bach_order.rb", + "test/test_base.rb", + "test/test_batch_iterator.rb", + "test/test_configuration.rb", + "test/test_formatters.rb", "unipept.gemspec" ] s.homepage = "https://github.com/unipept/unipept-cli/" s.licenses = ["MIT"] + s.require_paths = ["lib"] s.required_ruby_version = Gem::Requirement.new(">= 1.9.3") - s.rubygems_version = "2.2.2" + s.rubygems_version = "2.1.11" s.summary = "Command line interface to Unipept web services." if s.respond_to? :specification_version then @@ -63,6 +85,7 @@ Gem::Specification.new do |s| s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) + s.add_development_dependency(%q, [">= 0"]) else s.add_dependency(%q, ["~> 2.7"]) s.add_dependency(%q, ["~> 0.6"]) @@ -70,6 +93,7 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) end else s.add_dependency(%q, ["~> 2.7"]) @@ -78,6 +102,7 @@ Gem::Specification.new do |s| s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) end end