-
Notifications
You must be signed in to change notification settings - Fork 30
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Handle empty response body case #112
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -289,6 +289,7 @@ def do_get(url, params=nil, &block) | |||||||||||||||||||||||||||||||||||||||||||||||||||||
if block | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
current_total_chunk_size = 0 | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
response = client.get(target, params, header) {|res, chunk| | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
break if res.status != 200 | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
current_total_chunk_size += chunk.bytesize | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
block.call(res, chunk, current_total_chunk_size) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
@@ -300,17 +301,17 @@ def do_get(url, params=nil, &block) | |||||||||||||||||||||||||||||||||||||||||||||||||||||
else | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
response = client.get(target, params, header) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||
validate_content_length!(response, response.body.bytesize) if @ssl | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
end | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
status = response.code | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
# retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
$stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..." | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
sleep retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
cumul_retry_delay += retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
retry_delay *= 2 | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
redo # restart from beginning of do-while loop | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Original code had There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Original code also had Why not retry if block given is because if block given the block may be not idempotent. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah I meant if-else for td-client-ruby/lib/td/client/api.rb Lines 289 to 314 in e21f4d4
if block
...
else
...
end
if !block_given? && ..
...
redo
end changed to if block
...
else
...
if !block_given? && ...
...
redo
end
end There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ah nvm, sorry I got wrong about it. Behavior didn't changed |
||||||||||||||||||||||||||||||||||||||||||||||||||||||
end | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||
status = response.code | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
# retry if the HTTP error code is 500 or higher and we did not run out of retrying attempts | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
if !block_given? && status >= 500 && cumul_retry_delay < @max_cumul_retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
$stderr.puts "Error #{status}: #{get_error(response)}. Retrying after #{retry_delay} seconds..." | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
sleep retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
cumul_retry_delay += retry_delay | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
retry_delay *= 2 | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
redo # restart from beginning of do-while loop | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
validate_content_length!(response, response.body.to_s.bytesize) if @ssl | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
end | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
SystemCallError, OpenSSL::SSL::SSLError, SocketError, HTTPClient::TimeoutError, | ||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -272,6 +272,22 @@ def validate_content_length_with_range(response, current_total_chunk_size) | |
end | ||
end | ||
|
||
def validate_response_status(res, current_total_chunk_size=0) | ||
case res.status | ||
when 200 | ||
if current_total_chunk_size != 0 | ||
# try to resume but the server returns 200 | ||
raise_error("Get job result failed", res) | ||
end | ||
when 206 # resuming | ||
else | ||
if res.status/100 == 5 | ||
raise HTTPServerException | ||
end | ||
raise_error("Get job result failed", res) | ||
end | ||
end | ||
|
||
class HTTPServerException < StandardError | ||
end | ||
|
||
|
@@ -297,43 +313,28 @@ def job_result_download(job_id, format='msgpack', autodecode=true) | |
current_total_chunk_size = 0 | ||
infl = nil | ||
begin # LOOP of Network/Server errors | ||
response = nil | ||
client.get(url, params, header) do |res, chunk| | ||
unless response | ||
case res.status | ||
when 200 | ||
if current_total_chunk_size != 0 | ||
# try to resume but the server returns 200 | ||
raise_error("Get job result failed", res) | ||
end | ||
when 206 # resuming | ||
else | ||
if res.status/100 == 5 | ||
raise HTTPServerException | ||
end | ||
raise_error("Get job result failed", res) | ||
end | ||
if infl.nil? && autodecode | ||
case res.header['Content-Encoding'][0].to_s.downcase | ||
when 'gzip' | ||
infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16) | ||
when 'deflate' | ||
infl = Zlib::Inflate.new | ||
end | ||
response = client.get(url, params, header) do |res, chunk| | ||
validate_response_status(res, current_total_chunk_size) | ||
if infl.nil? && autodecode | ||
case res.header['Content-Encoding'][0].to_s.downcase | ||
when 'gzip' | ||
infl = Zlib::Inflate.new(Zlib::MAX_WBITS + 16) | ||
when 'deflate' | ||
infl = Zlib::Inflate.new | ||
end | ||
end | ||
response = res | ||
current_total_chunk_size += chunk.bytesize | ||
chunk = infl.inflate(chunk) if infl | ||
yield chunk, current_total_chunk_size | ||
end | ||
validate_response_status(response) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This line is intended for empty response body, so |
||
|
||
# completed? | ||
validate_content_length_with_range(response, current_total_chunk_size) | ||
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Timeout::Error, EOFError, | ||
SystemCallError, OpenSSL::SSL::SSLError, SocketError, HTTPClient::TimeoutError, | ||
HTTPServerException => e | ||
if response # at least a chunk is downloaded | ||
if current_total_chunk_size > 0 | ||
if etag = response.header['ETag'][0] | ||
header['If-Range'] = etag | ||
header['Range'] = "bytes=#{current_total_chunk_size}-" | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is in
else
forif block
, so!block_given?
is always true. We don't need to check it.