Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Imported coursera-facing files from rag

  • Loading branch information...
commit cb700aaa2f6d94251b8c65d4302875595bf58ee0 1 parent 377ca87
Richard Xia authored
View
9 config/autograders.yml.example
@@ -0,0 +1,9 @@
+# Format:
+# coursera-queue-name:
+# cmd: "command to grade assignment"
+#
+# Use a percent sign (%) to indicate the name of a file containing the user's
+# submission
+
+assignment-1-part-1:
+ cmd: "./grade % other-arguments"
View
9 config/conf.yml.example
@@ -0,0 +1,9 @@
+# Default configuration name
+default: saas-staging
+# Each configuration must have endpoint_uri, api_key, and autograders_yml
+saas-staging:
+ endpoint_uri: https://berkeley.campus-class.org/saas-staging/
+ api_key:
+ autograders_yml: autograders.yml
+ halt: true # default: true, exit when all submission queues are empty
+ sleep_duration: 300 # default 300, time in seconds to sleep when all queues are empty, only valid when halt == false
View
125 lib/auto_grader_subprocess.rb
@@ -0,0 +1,125 @@
+require 'tempfile'
+require 'open3'
+require 'timeout'
+
+require_relative 'rag_logger'
+
+module AutoGraderSubprocess
+ extend RagLogger
+ class AutoGraderSubprocess::OutputParseError < StandardError ; end
+ class AutoGraderSubprocess::SubprocessError < StandardError ; end
+
+ # FIXME: This is a hack, remove later
+ # This, and run_autograder, should really be part of a different module/class
+ # Runs a separate process for grading
+ def self.run_autograder_subprocess(submission, spec, grader_type)
+ stdout_text = stderr_text = nil
+ exitstatus = 0
+ Tempfile.open(['test', '.rb']) do |file|
+ file.write(submission)
+ file.flush
+
+ opts = {
+ :timeout => 60,
+ :cmd => %Q{./grade "#{file.path}" "#{spec}"}
+ }.merge case grader_type
+ when 'HerokuRspecGrader'
+ { :timeout => 180,
+ :cmd => %Q{./grade_heroku "#{submission}" "#{spec}"}
+ }
+ when 'HW3Grader'
+ {
+ :timeout => 400,
+ :cmd => %Q{./grade3 -a ../rottenpotatoes "#{file.path}" "#{spec}"}
+ }
+ when 'HW4Grader'
+ {
+ :timeout => 300,
+ :cmd => %Q{./grade4 "#{file.path}" "#{spec}"}
+ }
+ when 'ManualGrader'
+ {
+ :timeout => 300,
+ :cmd => %Q{./grade5 "#{file.path}"}
+ }
+ else
+ {}
+ end
+
+ begin
+ Timeout::timeout(opts[:timeout]) do
+ Open3.popen3 opts[:cmd] do |stdin, stdout, stderr, wait_thr|
+ if grader_type == 'ManualGrader'
+ # FIXME: This is really hacky
+ last_iteration = true
+ while (thread_alive = wait_thr.alive?) or last_iteration
+ begin
+ stdout_text = stdout.read_nonblock 1024
+ rescue Errno::EAGAIN => e
+ else
+ print stdout_text
+ end
+
+ begin
+ stdin_text = STDIN.read_nonblock 1024
+ rescue Errno::EAGAIN => e
+ else
+ stdin.write(stdin_text)
+ end
+ sleep(0.05)
+ last_iteration = false unless thread_alive
+ end
+ else
+ stdout_text = stdout.read; stderr_text = stderr.read
+ stdin.close; stdout.close; stderr.close
+ exitstatus = wait_thr.value.exitstatus
+ end
+ end
+ end
+ rescue Timeout::Error => e
+ exitstatus = -1
+ stderr_text = "Program timed out"
+ end
+
+ if exitstatus != 0
+ logger.fatal "AutograderSubprocess error: #{stderr_text}"
+ raise AutoGraderSubprocess::SubprocessError, "AutograderSubprocess error: #{stderr_text}"
+ end
+ end
+ score, comments = parse_grade(stdout_text)
+ comments.gsub!(spec, 'spec.rb')
+ [score, comments]
+ rescue ArgumentError => e
+ logger.error e.to_s
+ score = 0
+ comments = e.to_s
+ [score, comments]
+ end
+
+ def run_autograder_subprocess(submission, spec, grader_type)
+ AutoGraderSubprocess.run_autograder_subprocess(submission, spec, grader_type)
+ end
+
+ # FIXME: This is related to the below hack, remove later
+ def self.parse_grade(str)
+ # Used for parsing the stdout output from running grade as a shell command
+ # FIXME: This feels insecure and fragile
+ score_regex = /Score out of \d+:\s*(\d+(?:\.\d+)?)$/
+ score = str.match(score_regex, str.rindex(score_regex))[1].to_f
+ comments = str.match(/^---BEGIN (?:cucumber|rspec|grader) comments---\n#{'-'*80}\n(.*)#{'-'*80}\n---END (?:cucumber|rspec|grader) comments---$/m)[1]
+ comments = comments.split("\n").map do |line|
+ line.gsub(/\(FAILED - \d+\)/, "(FAILED)")
+ end.join("\n")
+ [score, comments]
+ rescue ArgumentError => e
+ logger.error "Error running parse_grade: #{e.to_s}; #{str}"
+ [0, e.to_s]
+ rescue StandardError => e
+ logger.fatal "Failed to parse autograder output: #{str}"
+ raise OutputParseError, "Failed to parse autograder output: #{str}"
+ end
+
+ def parse_grade(str)
+ AutoGraderSubprocess.parse_grade(str)
+ end
+end
View
214 lib/coursera_client.rb
@@ -0,0 +1,214 @@
+require 'tempfile'
+require 'yaml'
+require 'net/http'
+require 'base64'
+
+require_relative 'rag_logger'
+require_relative 'coursera_controller'
+require_relative 'coursera_submission'
+require_relative 'auto_grader'
+require_relative 'auto_grader_subprocess'
+
+class CourseraClient
+ include RagLogger
+ include AutoGraderSubprocess
+
+ class CourseraClient::UnknownAssignmentPart < StandardError ; end
+ class CourseraClient::SpecNotFound < StandardError ; end
+
+ # Requires a file called 'autograders.yml' to exist in the current working
+ # directory and it must represent a hash from assignment_part_sid's to
+ # spec URIs
+
+ #def initialize(endpoint, api_key, autograders_yml)
+ def initialize(conf_name=nil)
+ conf = load_configurations(conf_name)
+
+ @endpoint = conf['endpoint_uri']
+ @api_key = conf['api_key']
+ @controller = CourseraController.new(@endpoint, @api_key)
+ @halt = conf['halt']
+ @sleep_duration = conf['sleep_duration'].nil? ? 5*60 : conf['sleep_duration'] # in seconds
+
+ # Load configuration file for assignment_id->spec map
+ # We assume that the keys are also the assignment_part_sids, as well as the queue_ids
+ @autograders = init_autograders(conf['autograders_yml'])
+ end
+
+ def run
+ each_submission do |assignment_part_sid, result|
+ submission = decode_submission(result)
+ spec = load_spec(assignment_part_sid)
+ grader_type = @autograders[assignment_part_sid][:type]
+
+ # FIXME: Use non-subprocess version instead
+ begin
+ score, comments = run_autograder_subprocess(submission, spec, grader_type) # defined in AutoGraderSubprocess
+ rescue AutoGraderSubprocess::SubprocessError => e
+ score = 0
+ comments = e.to_s
+ rescue AutoGraderSubprocess::OutputParseError => e
+ score = 0
+ comments = e.to_s
+ rescue
+ logger.fatal(submission)
+ raise
+ end
+ formatted_comments = format_for_html(comments)
+ @controller.post_score(result['api_state'], score, formatted_comments)
+ logger.debug " scored #{score}: #{comments}"
+ end
+ rescue Exception => e
+ logger.fatal(e)
+ raise
+ end
+
+ def download_submissions(file)
+ # Iterate through assignment parts until all queues are empty
+ # Note, this process MUST complete in under 15 minutes or else the queues
+ # will start repopulating. This method does NOT permanently remove
+ # submissions from the queue
+ if file.nil?
+ logger.fatal 'Target file is nil'
+ end
+ submissions = {}
+ @autograders.each_key {|x| submissions[x] = []}
+
+ @autograders.keys.each do |assignment_part_sid|
+ while true
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " deleting assignment part"
+ break
+ end
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " got submission"
+ submissions[assignment_part_sid] << result
+ end
+ end
+ logger.info "Finishing"
+ file.write(submissions.inspect)
+ file.flush
+ end
+
+ private
+
+ def load_spec(assignment_part_sid)
+ unless @autograders.include?(assignment_part_sid)
+ logger.fatal "Assignment part #{assignment_part_sid} not found!"
+ raise "Assignment part #{assignment_part_sid} not found!"
+ end
+ autograder = @autograders[assignment_part_sid]
+ return autograder[:uri] if autograder[:uri] !~ /^http/ # Assume that if uri doesn't start with http, then it is a local file path
+
+ # If not in cache, download and add to cache
+ if autograder[:cache].nil?
+ spec_file = Tempfile.new('spec')
+ response = Net::HTTP.get_response(URI(autograder[:uri]))
+ if response.code !~ /2\d\d/
+ logger.fatal "Could not load the spec at #{autograder[:uri]}"
+ raise CourseraClient::SpecNotFound, "Could not load the spec at #{autograder[:uri]}"
+ end
+ spec_file.write(response.body)
+ spec_file.close
+ autograder[:cache] = spec_file
+ end
+ autograder[:cache].path
+ end
+
+ def run_autograder(submission, spec, grader_type)
+ g = AutoGrader.create('1', grader_type, submission, :spec => spec)
+ g.grade!
+ g
+ end
+
+ # Returns hash of assignment_part_ids to hashes containing uri and grader type
+ # i.e. { "assign-1-part-1" => {:uri => 'http://example.com', :type => 'RspecGrader' } }
+ def init_autograders(filename)
+ # TODO: Verify file format
+ yml = YAML::load(File.open(filename, 'r'))
+ yml.each_pair do |id, obj|
+ # Convert keys from string to sym
+ yml[id] = obj.inject({}){|memo, (k,v)| memo[k.to_sym] = v; memo}
+ end
+ end
+
+ # Formats autograder ouput for display in browser
+ def format_for_html(text)
+ "<pre>#{text.gsub(/&/, '&amp;').gsub(/</, '&lt;').gsub(/>/, '&gt;')}</pre>" # sanitize html
+ # .gsub(/^( +)/){|s| "&nbsp;"*s.size} # indentation
+ # .gsub(/\n/, "<br />\n") # newlines
+ end
+
+ def decode_submission(submission)
+ case submission['submission_encoding']
+ when 'base64'
+ Base64.strict_decode64(submission['submission'])
+ else
+ logger.fatal "Can't handle encoding: #{submission['submission_encoding']}"
+ raise "Can't handle encoding: #{submission['submission_encoding']}"
+ end
+ end
+
+ def each_submission
+ if @halt
+ # Iterate round robin through assignment parts until all queues are empty
+ # parameterize this differently
+ while @autograders.size > 0
+ to_delete = []
+ @autograders.keys.each do |assignment_part_sid|
+ logger.info assignment_part_sid
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " queue length 0; removing"
+ to_delete << assignment_part_sid
+ next
+ end
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " received submission: #{result['submission_metadata']['submission_id']}"
+ logger.debug result['submission_metadata']
+
+ yield assignment_part_sid, result
+ end
+ @autograders.delete_if{|key,value| to_delete.include? key}
+ end
+ else
+
+ # Loop forever
+ while true
+ all_empty = true
+ @autograders.keys.each do |assignment_part_sid|
+ logger.info assignment_part_sid
+ if @controller.get_queue_length(assignment_part_sid) == 0
+ logger.info " queue length 0"
+ next
+ end
+ all_empty = false
+ result = @controller.get_pending_submission(assignment_part_sid)
+ next if result.nil?
+ logger.info " received submission: #{result['submission_metadata']['submission_id']}"
+ logger.debug result['submission_metadata']
+
+ yield assignment_part_sid, result
+ end
+ if all_empty
+ logger.info "sleeping for #{@sleep_duration} seconds"
+ sleep @sleep_duration
+ end
+ end
+ end
+ end
+
+ def load_configurations(conf_name=nil)
+ config_path = 'config/conf.yml'
+ unless File.file?(config_path)
+ puts "Please copy conf.yml.example into conf.yml and configure the parameters"
+ exit
+ end
+ confs = YAML::load(File.open(config_path, 'r'){|f| f.read})
+ conf_name ||= confs['default'] || confs.keys.first
+ conf = confs[conf_name]
+ raise "Couldn't load configuration #{conf_name}" if conf.nil?
+ conf
+ end
+end
View
118 lib/coursera_controller.rb
@@ -0,0 +1,118 @@
+require_relative 'rag_logger'
+
+class CourseraController
+ include RagLogger
+
+ class CourseraController::InvalidHTTPMethodError < StandardError ; end
+ class CourseraController::BadStatusCodeError < StandardError ; end
+
+ require 'net/https'
+ require 'json'
+
+ attr_accessor :base_uri
+
+ HTTP_MODES = [:get, :post]
+
+ def initialize(base_uri, api_key)
+ @base_uri = base_uri
+ @api_key = api_key # This can be found on the Course Settings page
+ end
+
+ # The response for this is currently undefined
+ def get_user_info(user_id)
+ params = {:user_id => user_id}
+ response = send_request("assignment/api/user_info/", params, :get)
+ response
+ end
+
+ # Returns length of queue
+ # Beware of concurrency: just because you receive a non-zero return value
+ # from get_queue_length() doesn't mean you are guaranteed to receive a
+ # submission from get_pending_submission()
+ def get_queue_length(queue_name)
+ begin
+ params = {:queue => queue_name}
+ response = send_request("assignment/api/queue_length/", params, :get)
+ if response['status'] =~ /[3-5]\d\d/
+ logger.error "Bad queue length response: #{response['status']}"
+ raise CourseraController::BadStatusCodeError, "Bad queue length response: #{response['status']}"
+ end
+ response['queue_length']
+ #rescue EOFError => e
+ # logger.error e.to_s
+ # logger.error "Retrying"
+ # retry
+ end
+ end
+
+ # Returns either nil or
+ # {
+ # "api_state": ...
+ # "user_info": ...
+ # "submission_metadata": {“id”: X, ...}
+ # "solutions": ...
+ # "submission_encoding": ... (will be set to ‘base64’)
+ # "submission": (actual user submission in Base64 format) }
+ # }
+ def get_pending_submission(queue_name, peek=nil)
+ params = {:queue => queue_name}
+ params[:peek] = true if peek
+ response = send_request("assignment/api/pending_submission/", params, :get)
+ response['submission']
+ end
+
+ # This is untested
+ def get_submission(submission_id)
+ params = {:submission_id => submission_id}
+ response = send_request("assignment/api/submission/", params, :get)
+ response['submission']
+ end
+
+ # Raises BadStatusCode if response status isn't 2xx (success)
+ def post_score(api_state, score, feedback="", options={})
+ params = {:api_state => api_state, :score => score, :feedback => feedback, :options => options.to_json}
+ response = send_request("assignment/api/score/", params, :post)
+ if response['status'] !~ /2\d\d/
+ logger.error "Bad post score response: #{response['status']}"
+ raise CourseraController::BadStatusCodeError, "Bad post score response: #{response['status']}"
+ end
+ end
+
+ private
+
+ # Sends an HTTPS GET request to the application path with the provided parameters.
+ # Returns a Hash representing the JSON-encoded response.
+ def send_request(path, params={}, mode=:get)
+ unless HTTP_MODES.include? mode
+ logger.fatal "Invalid mode: #{mode}"
+ raise CourseraController::InvalidHTTPMethodError, "Invalid mode: #{mode}"
+ end
+ uri = URI.join(@base_uri, path)
+ uri.query = URI.encode_www_form(params) if mode == :get
+
+ http = Net::HTTP.new(uri.host, uri.port)
+ http.use_ssl = true
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+ request = Net::HTTP::Post.new(uri.request_uri)
+ request.set_form_data(params) if mode == :post
+ request['X-api-key'] = @api_key
+
+ logged_eof = false
+ begin
+ response = http.request(request)
+ rescue EOFError # invalid url?
+ sleep 10
+ logger.error "Invalid response (EOFError): #{path}" unless logged_eof
+ logged_eof = true
+ retry
+ end
+
+ begin
+ JSON.parse(response.body)
+ rescue StandardError => e
+ logger.error "Invalid JSON in response body #{response.body}"
+ raise e
+ end
+ end
+end
View
42 lib/coursera_submission.rb
@@ -0,0 +1,42 @@
+require_relative 'rag_logger'
+
+require 'base64'
+require 'json'
+class CourseraSubmission
+ # sid that corresponds to an assignment part on Coursera
+ attr_accessor :assignment_part_sid
+ # login email address of submitter
+ attr_accessor :email_address
+ # text contents of submission
+ attr_accessor :submission
+ # optional auxiliary text
+ attr_accessor :submission_aux
+
+ def initialize(assignment_part_sid, email_address, submission,
+ submission_aux="")
+ @assignment_part_sid = assignment_part_sid
+ @email_address = email_address
+ @submission = submission
+ @submission_aux = submission_aux
+ end
+
+ def self.load_from_base64(base64_text)
+ attrs = JSON.parse(Base64.strict_decode64(base64_text))
+ assignment_part_sid = Base64.strict_decode64(attrs['assignment_part_sid'])
+ email_address = Base64.strict_decode64(attrs['email_address'])
+ submission = Base64.strict_decode64(attrs['submission'])
+ submission_aux = Base64.strict_decode64(attrs['submission_aux'])
+ self.new(assignment_part_sid, email_address, submission, submission_aux)
+ end
+
+ def to_json
+ obj = {
+ :assignment_part_sid => Base64.strict_encode64(@assignment_part_sid),
+ :email_address => Base64.strict_encode64(@email_address),
+ :submission => Base64.strict_encode64(@submission),
+ :submission_aux => Base64.strict_encode64(@submission_aux),
+ }
+ obj.to_json
+ end
+end
+
View
15 lib/rag_logger.rb
@@ -0,0 +1,15 @@
+# http://stackoverflow.com/questions/917566/ruby-share-logger-instance-among-module-classes
+require 'logger'
+module RagLogger
+ def logger
+ RagLogger.logger
+ end
+
+ def self.logger
+ unless @logger_file
+ Dir.mkdir('log/') unless File.directory?('log/')
+ @logger_file ||= "log/rag-#{Process.pid}.log"
+ end
+ @logger ||= Logger.new(@logger_file, 0, 1024*1024)
+ end
+end
View
7 run_coursera_client.rb
@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require './lib/coursera_client.rb'
+
+# First argument is optional, name of the configuration profile
+
+CourseraClient.new(ARGV[0]).run

0 comments on commit cb700aa

Please sign in to comment.
Something went wrong with that request. Please try again.