-
Notifications
You must be signed in to change notification settings - Fork 8
/
s3_upload.rb
130 lines (108 loc) · 3.93 KB
/
s3_upload.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
require 'rubygems'
require 'xmlsimple'
require 'cgi'
require 'net/https'
def upload_results_to_s3 (server_name, job_id, type, fname, fpath_name)
s3_data = ''
response = Net::HTTP.get(URI.parse("http://#{server_name}/jobs/#{job_id}/uploadurls.xml?upload_type=#{type}&files=#{fname}"))
s3_data = XmlSimple.xml_in(response)
# puts s3_data.inspect
#this is just so we can parse it fast
host = s3_data["files"].first[fname].first['host'].first
port = s3_data["files"].first[fname].first['port'].first
header = s3_data["files"].first[fname].first['header'].first
ssl = s3_data["files"].first[fname].first['ssl'].first['content']
path = s3_data["files"].first[fname].first['path'].first
File.open(fpath_name, 'rb') do |up_file|
send_request('Put', host, port, path, header, up_file, ssl)
end
end
def upload_data_to_s3 (server_name, job_id, type, fname, fpath_name)
s3_data = ''
response = Net::HTTP.get(URI.parse("http://#{server_name}/jobs/#{job_id}/uploadurls.xml?upload_type=#{type}&files=#{fname}"))
s3_data = XmlSimple.xml_in(response)
#this is just so we can parse it fast
host = s3_data["files"].first[fname].first['host'].first
port = s3_data["files"].first[fname].first['port'].first
header = s3_data["files"].first[fname].first['header'].first
ssl = s3_data["files"].first[fname].first['ssl'].first['content']
path = s3_data["files"].first[fname].first['path'].first
File.open(fpath_name, 'rb') do |up_file|
send_request('Put', host, port, path, header, up_file, ssl)
end
#create dataset object form path
Net::HTTP.get(URI.parse("http://#{server_name}/data_sets/create_from_path.xml?name=#{fname}&job_id=#{job_id}&path=#{path}"))
end
def fetch_data_file(s3url, dest_file)
url = URI.parse s3url
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = (url.scheme == 'https')
url_fetch = url.path
url_fetch = url.path + '?' + url.query if !url.query.nil?
# save output
data_file_handle = File.new(dest_file, 'wb')
request = Net::HTTP::Get.new(url_fetch)
# stream the file (efficient for larger files
http.request(request) do |res|
size, total = 0, res.header['Content-Length'].to_i
res.read_body do |chunk|
size += chunk.size
#puts "%d%% done (%d of %d)" % [(size * 100) / total, size, total]
data_file_handle.write chunk
end
end
data_file_handle.close
end
# Helper to generate REST requests, handle authentication and errors
def send_request(req_type, host, port, url, fields_hash, up_file, ssl)
res = false
conn_start(host, port, ssl) do |http|
request = Module.module_eval("Net::HTTP::#{req_type}").new(url)
fields_hash.each_pair{|n,v| request[n] = v} # Add user defined header fields
request.content_length = up_file.lstat.size
request.body_stream = up_file # For uploads using streaming
begin
response = http.request(request)
rescue Exception => ex
response = ex.to_s
end
# Handle auth errors and other errors!
if !response.is_a?(Net::HTTPOK)
#puts 'Failed to upload!'
#log_request(request, response)
else
res = true
end
end
end
# Create a connection with all the needed setting
# And yeald to the block
def conn_start host, port, ssl
con = Net::HTTP.new(host, port)
if ssl
con.use_ssl = true
con.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
con.start
if block_given?
begin
return yield(con)
ensure
con.finish
end
end
con
end
# Display the request and response...
def log_request(request, response)
puts "\nRequest Headers"
request.each {|head,val| puts "#{head}: #{val}"}
puts "\nResponse Headers"
response.each {|head,val| puts "#{head}: #{val}"}
puts "\nResponse Body"
puts response.body
puts "\nResponse Type"
puts "#{response.class} (#{response.code})"
end
#upload_results_to_s3("crdata.org", 184, "logs", "job.log", "job.log")
#upload_results_to_s3("crdata.org", 184, "results", "index.html", "index.html")