Permalink
Browse files

Imported coursera-facing files from rag

  • Loading branch information...
1 parent 377ca87 commit cb700aaa2f6d94251b8c65d4302875595bf58ee0 Richard Xia committed May 23, 2012
@@ -0,0 +1,9 @@
+# Format:
+# coursera-queue-name:
+# cmd: "command to grade assignment"
+#
+# Use a percent sign (%) to indicate the name of a file containing the user's
+# submission
+
+assignment-1-part-1:
+ cmd: "./grade % other-arguments"
View
@@ -0,0 +1,9 @@
+# Default configuration name
+default: saas-staging
+# Each configuration must have endpoint_uri, api_key, and autograders_yml
+saas-staging:
+ endpoint_uri: https://berkeley.campus-class.org/saas-staging/
+ api_key:
+ autograders_yml: autograders.yml
+ halt: true # default: true, exit when all submission queues are empty
+ sleep_duration: 300 # default 300, time in seconds to sleep when all queues are empty, only valid when halt == false
@@ -0,0 +1,125 @@
+require 'tempfile'
+require 'open3'
+require 'timeout'
+
+require_relative 'rag_logger'
+
+module AutoGraderSubprocess
+ extend RagLogger
+ class AutoGraderSubprocess::OutputParseError < StandardError ; end
+ class AutoGraderSubprocess::SubprocessError < StandardError ; end
+
+ # FIXME: This is a hack, remove later
+ # This, and run_autograder, should really be part of a different module/class
+ # Runs a separate process for grading
+ def self.run_autograder_subprocess(submission, spec, grader_type)
+ stdout_text = stderr_text = nil
+ exitstatus = 0
+ Tempfile.open(['test', '.rb']) do |file|
+ file.write(submission)
+ file.flush
+
+ opts = {
+ :timeout => 60,
+ :cmd => %Q{./grade "#{file.path}" "#{spec}"}
+ }.merge case grader_type
+ when 'HerokuRspecGrader'
+ { :timeout => 180,
+ :cmd => %Q{./grade_heroku "#{submission}" "#{spec}"}
+ }
+ when 'HW3Grader'
+ {
+ :timeout => 400,
+ :cmd => %Q{./grade3 -a ../rottenpotatoes "#{file.path}" "#{spec}"}
+ }
+ when 'HW4Grader'
+ {
+ :timeout => 300,
+ :cmd => %Q{./grade4 "#{file.path}" "#{spec}"}
+ }
+ when 'ManualGrader'
+ {
+ :timeout => 300,
+ :cmd => %Q{./grade5 "#{file.path}"}
+ }
+ else
+ {}
+ end
+
+ begin
+ Timeout::timeout(opts[:timeout]) do
+ Open3.popen3 opts[:cmd] do |stdin, stdout, stderr, wait_thr|
+ if grader_type == 'ManualGrader'
+ # FIXME: This is really hacky
+ last_iteration = true
+ while (thread_alive = wait_thr.alive?) or last_iteration
+ begin
+ stdout_text = stdout.read_nonblock 1024
+ rescue Errno::EAGAIN => e
+ else
+ print stdout_text
+ end
+
+ begin
+ stdin_text = STDIN.read_nonblock 1024
+ rescue Errno::EAGAIN => e
+ else
+ stdin.write(stdin_text)
+ end
+ sleep(0.05)
+ last_iteration = false unless thread_alive
+ end
+ else
+ stdout_text = stdout.read; stderr_text = stderr.read
+ stdin.close; stdout.close; stderr.close
+ exitstatus = wait_thr.value.exitstatus
+ end
+ end
+ end
+ rescue Timeout::Error => e
+ exitstatus = -1
+ stderr_text = "Program timed out"
+ end
+
+ if exitstatus != 0
+ logger.fatal "AutograderSubprocess error: #{stderr_text}"
+ raise AutoGraderSubprocess::SubprocessError, "AutograderSubprocess error: #{stderr_text}"
+ end
+ end
+ score, comments = parse_grade(stdout_text)
+ comments.gsub!(spec, 'spec.rb')
+ [score, comments]
+ rescue ArgumentError => e
+ logger.error e.to_s
+ score = 0
+ comments = e.to_s
+ [score, comments]
+ end
+
+ def run_autograder_subprocess(submission, spec, grader_type)
+ AutoGraderSubprocess.run_autograder_subprocess(submission, spec, grader_type)
+ end
+
+ # FIXME: This is related to the below hack, remove later
+ def self.parse_grade(str)
+ # Used for parsing the stdout output from running grade as a shell command
+ # FIXME: This feels insecure and fragile
+ score_regex = /Score out of \d+:\s*(\d+(?:\.\d+)?)$/
+ score = str.match(score_regex, str.rindex(score_regex))[1].to_f
+ comments = str.match(/^---BEGIN (?:cucumber|rspec|grader) comments---\n#{'-'*80}\n(.*)#{'-'*80}\n---END (?:cucumber|rspec|grader) comments---$/m)[1]
+ comments = comments.split("\n").map do |line|
+ line.gsub(/\(FAILED - \d+\)/, "(FAILED)")
+ end.join("\n")
+ [score, comments]
+ rescue ArgumentError => e
+ logger.error "Error running parse_grade: #{e.to_s}; #{str}"
+ [0, e.to_s]
+ rescue StandardError => e
+ logger.fatal "Failed to parse autograder output: #{str}"
+ raise OutputParseError, "Failed to parse autograder output: #{str}"
+ end
+
+ def parse_grade(str)
+ AutoGraderSubprocess.parse_grade(str)
+ end
+end
View
@@ -0,0 +1,214 @@
+require 'tempfile'
+require 'yaml'
+require 'net/http'
+require 'base64'
+
+require_relative 'rag_logger'
+require_relative 'coursera_controller'
+require_relative 'coursera_submission'
+require_relative 'auto_grader'
+require_relative 'auto_grader_subprocess'
+
+class CourseraClient
+ include RagLogger
+ include AutoGraderSubprocess
+
+ class CourseraClient::UnknownAssignmentPart < StandardError ; end
+ class CourseraClient::SpecNotFound < StandardError ; end
+
+ # Requires a file called 'autograders.yml' to exist in the current working
+ # directory and it must represent a hash from assignment_part_sid's to
+ # spec URIs
+
+ #def initialize(endpoint, api_key, autograders_yml)
+ def initialize(conf_name=nil)
+ conf = load_configurations(conf_name)
+
+ @endpoint = conf['endpoint_uri']
+ @api_key = conf['api_key']
+ @controller = CourseraController.new(@endpoint, @api_key)
+ @halt = conf['halt']
+ @sleep_duration = conf['sleep_duration'].nil? ? 5*60 : conf['sleep_duration'] # in seconds
+
+ # Load configuration file for assignment_id->spec map
+ # We assume that the keys are also the assignment_part_sids, as well as the queue_ids
+ @autograders = init_autograders(conf['autograders_yml'])
+ end
+
+ def run
+ each_submission do |assignment_part_sid, result|
+ submission = decode_submission(result)
+ spec = load_spec(assignment_part_sid)
+ grader_type = @autograders[assignment_part_sid][:type]
+
+ # FIXME: Use non-subprocess version instead
+ begin
+ score, comments = run_autograder_subprocess(submission, spec, grader_type) # defined in AutoGraderSubprocess
+ rescue AutoGraderSubprocess::SubprocessError => e
+ score = 0
+ comments = e.to_s
+ rescue AutoGraderSubprocess::OutputParseError => e
+ score = 0
+ comments = e.to_s
+ rescue
+ logger.fatal(submission)
+ raise
+ end
+ formatted_comments = format_for_html(comments)
+ @controller.post_score(result['api_state'], score, formatted_comments)
+ logger.debug " scored #{score}: #{comments}"
+ end
+ rescue Exception => e
+ logger.fatal(e)
+ raise
+ end
+
+ def download_submissions(file)
+ # Iterate through assignment parts until all queues are empty
+ # Note, this process MUST complete in under 15 minutes or else the queues
+ # will start repopulating. This method does NOT permanently remove
+ # submissions from the queue
+ if file.nil?
+ logger.fatal 'Target file is nil'
+ end
+ submissions = {}
+ @autograders.each_key {|x| submissions[x] = []}
+
+ @autograders.keys.each do |assignment_part_sid|
+ while true
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " deleting assignment part"
+ break
+ end
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " got submission"
+ submissions[assignment_part_sid] << result
+ end
+ end
+ logger.info "Finishing"
+ file.write(submissions.inspect)
+ file.flush
+ end
+
+ private
+
+ def load_spec(assignment_part_sid)
+ unless @autograders.include?(assignment_part_sid)
+ logger.fatal "Assignment part #{assignment_part_sid} not found!"
+ raise "Assignment part #{assignment_part_sid} not found!"
+ end
+ autograder = @autograders[assignment_part_sid]
+ return autograder[:uri] if autograder[:uri] !~ /^http/ # Assume that if uri doesn't start with http, then it is a local file path
+
+ # If not in cache, download and add to cache
+ if autograder[:cache].nil?
+ spec_file = Tempfile.new('spec')
+ response = Net::HTTP.get_response(URI(autograder[:uri]))
+ if response.code !~ /2\d\d/
+ logger.fatal "Could not load the spec at #{autograder[:uri]}"
+ raise CourseraClient::SpecNotFound, "Could not load the spec at #{autograder[:uri]}"
+ end
+ spec_file.write(response.body)
+ spec_file.close
+ autograder[:cache] = spec_file
+ end
+ autograder[:cache].path
+ end
+
+ def run_autograder(submission, spec, grader_type)
+ g = AutoGrader.create('1', grader_type, submission, :spec => spec)
+ g.grade!
+ g
+ end
+
+ # Returns hash of assignment_part_ids to hashes containing uri and grader type
+ # i.e. { "assign-1-part-1" => {:uri => 'http://example.com', :type => 'RspecGrader' } }
+ def init_autograders(filename)
+ # TODO: Verify file format
+ yml = YAML::load(File.open(filename, 'r'))
+ yml.each_pair do |id, obj|
+ # Convert keys from string to sym
+ yml[id] = obj.inject({}){|memo, (k,v)| memo[k.to_sym] = v; memo}
+ end
+ end
+
+ # Formats autograder ouput for display in browser
+ def format_for_html(text)
+ "<pre>#{text.gsub(/&/, '&amp;').gsub(/</, '&lt;').gsub(/>/, '&gt;')}</pre>" # sanitize html
+ # .gsub(/^( +)/){|s| "&nbsp;"*s.size} # indentation
+ # .gsub(/\n/, "<br />\n") # newlines
+ end
+
+ def decode_submission(submission)
+ case submission['submission_encoding']
+ when 'base64'
+ Base64.strict_decode64(submission['submission'])
+ else
+ logger.fatal "Can't handle encoding: #{submission['submission_encoding']}"
+ raise "Can't handle encoding: #{submission['submission_encoding']}"
+ end
+ end
+
+ def each_submission
+ if @halt
+ # Iterate round robin through assignment parts until all queues are empty
+ # parameterize this differently
+ while @autograders.size > 0
+ to_delete = []
+ @autograders.keys.each do |assignment_part_sid|
+ logger.info assignment_part_sid
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " queue length 0; removing"
+ to_delete << assignment_part_sid
+ next
+ end
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " received submission: #{result['submission_metadata']['submission_id']}"
+ logger.debug result['submission_metadata']
+
+ yield assignment_part_sid, result
+ end
+ @autograders.delete_if{|key,value| to_delete.include? key}
+ end
+ else
+
+ # Loop forever
+ while true
+ all_empty = true
+ @autograders.keys.each do |assignment_part_sid|
+ logger.info assignment_part_sid
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " queue length 0"
+ next
+ end
+ all_empty = false
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " received submission: #{result['submission_metadata']['submission_id']}"
+ logger.debug result['submission_metadata']
+
+ yield assignment_part_sid, result
+ end
+ if all_empty
+ logger.info "sleeping for #{@sleep_duration} seconds"
+ sleep @sleep_duration
+ end
+ end
+ end
+ end
+
+ def load_configurations(conf_name=nil)
+ config_path = 'config/conf.yml'
+ unless File.file?(config_path)
+ puts "Please copy conf.yml.example into conf.yml and configure the parameters"
+ exit
+ end
+ confs = YAML::load(File.open(config_path, 'r'){|f| f.read})
+ conf_name ||= confs['default'] || confs.keys.first
+ conf = confs[conf_name]
+ raise "Couldn't load configuration #{conf_name}" if conf.nil?
+ conf
+ end
+end
Oops, something went wrong.

0 comments on commit cb700aa

Please sign in to comment.