Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

added support to upload data files from R scripts, the data files mus…

…t be .sav,.out or .crd
  • Loading branch information...
commit 9e82f96f95da8a080d70a1ebbddf1766db35cbbe 1 parent 027838c
@rdulepet authored
Showing with 53 additions and 0 deletions.
  1. +27 −0 job.rb
  2. +3 −0  processing_node.rb
  3. +23 −0 s3_upload.rb
View
27 job.rb
@@ -30,6 +30,7 @@ class Job
attr_reader :r_script_filename, :job_id, :curr_uuid, :r_call_interface
attr_reader :job_status, :r_script_inc_filename, :doc
attr_reader :server_node
+ attr_reader :in_data_files
def initialize(xml_response, server_node)
#@r_call_interface = RSRuby.instance
@@ -39,6 +40,7 @@ def initialize(xml_response, server_node)
@curr_uuid = Global.rand_uuid
@job_status = Global::FAILED_JOB
@server_node = server_node
+ @in_data_files = Hash.new
# log request
Global.logger.info(xml_response)
@@ -96,6 +98,9 @@ def fetch_params
#@r_call_interface.assign(job_params[PARAM_NAME], just_name)
r_script_inc_file_handle.puts "#{job_params[PARAM_NAME]} = \"#{just_name}\""
Global.logger.info("R_PARAMETER::#{job_params[PARAM_NAME]} = #{just_name}")
+
+ # track all input data files
+ @in_data_files[just_name] = 1
fetch_data_file job_params[PARAM_DATA_SET], "#{Global.results_dir}/#{@curr_uuid}/#{just_name}"
elsif job_params[PARAM_KIND] == VALUE_STRING
@@ -214,4 +219,26 @@ def store_results_and_logs
"#{Global.results_dir}/#{@curr_uuid}/#{name}")
}
end
+
+ def store_data
+ # now iterate through directory and store all results files (web content only)
+
+ # upload only web content files for results
+ # .html,.htm,.css,.png,.pdf,.jpg
+ # iterate through directory and store files one at a time in S3
+ upload_files = Dir[File.join("#{Global.results_dir}/#{@curr_uuid}", "*")].select{|file| File.ftype(file) == "file" &&
+ (File.extname(file) == '.crd' ||
+ File.extname(file) == '.out' ||
+ File.extname(file) == '.sav')}.each{|name|
+ name = name.split("/").last
+ if ! @in_data_files.has_key?(name)
+ puts "DATA_OUTPUT_FILE = #{Global.results_dir}/#{@curr_uuid}/#{name}"
+ upload_data_to_s3(@server_node,
+ @job_id,
+ "data",
+ name,
+ "#{Global.results_dir}/#{@curr_uuid}/#{name}")
+ end
+ }
+ end
end
View
3  processing_node.rb
@@ -61,6 +61,9 @@ def run
# STEP 7
job.store_results_and_logs if !job.nil?
+ # STEP 7
+ job.store_data if !job.nil?
+
# STEP 8
job_completed(job) if !job.nil?
rescue => err
View
23 s3_upload.rb
@@ -23,6 +23,29 @@ def upload_results_to_s3 (server_name, job_id, type, fname, fpath_name)
end
+
+def upload_data_to_s3 (server_name, job_id, type, fname, fpath_name)
+ s3_data = ''
+ response = Net::HTTP.get(URI.parse("http://#{server_name}/jobs/#{job_id}/uploadurls.xml?upload_type=#{type}&files=#{fname}"))
+
+ s3_data = XmlSimple.xml_in(response)
+ #puts s3_data.inspect
+
+ #this is just so we can parse it fast
+ host = s3_data["files"].first[fname].first['host'].first
+ port = s3_data["files"].first[fname].first['port'].first
+ header = s3_data["files"].first[fname].first['header'].first
+ ssl = s3_data["files"].first[fname].first['ssl'].first['content']
+ path = s3_data["files"].first[fname].first['path'].first
+
+ File.open(fpath_name, 'rb') do |up_file|
+ send_request('Put', host, port, path, header, up_file, ssl)
+ end
+
+ #create dataset object form path
+ Net::HTTP.get(URI.parse("http://#{server_name}/data_sets/create_from_path.xml?name=job_result_dataset&job_id=#{job_id}&path=#{path}"))
+end
+
def fetch_data_file(s3url, dest_file)
url = URI.parse s3url
http = Net::HTTP.new(url.host, url.port)

0 comments on commit 9e82f96

Please sign in to comment.
Something went wrong with that request. Please try again.