Version on the ec2 graders

autograders3.yml

@@ -0,0 +1,37 @@
+# This does not support http URIs yet
+
+# assign-1-part-1:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part1_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-2:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part2_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-3:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part3_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-4:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part4_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-5:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part5_spec.rb
+#   type: WeightedRspecGrader
+# 
+# assign-2-part-1:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part1_spec.rb
+#   type: WeightedRspecGrader
+# assign-2-part-2:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part2_spec.rb
+#   type: WeightedRspecGrader
+# assign-2-part-3:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part3_spec.rb
+#   type: HerokuRspecGrader
+# assign-2-part-4:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part4_spec.rb
+#   type: HerokuRspecGrader
+# assign-2-part-5:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part5_spec.rb
+#   type: HerokuRspecGrader
+
+assign-3-part-1:
+  uri: hw3.yml
+  type: HW3Grader

config/autograders.yml

@@ -1,33 +1,37 @@
 # This does not support http URIs yet
 
-assign-1-part-1:
-  uri: http://s3.amazonaws.com/saasbook/assign1/part1_spec.rb
-  type: WeightedRspecGrader
-assign-1-part-2:
-  uri: http://s3.amazonaws.com/saasbook/assign1/part2_spec.rb
-  type: WeightedRspecGrader
-assign-1-part-3:
-  uri: http://s3.amazonaws.com/saasbook/assign1/part3_spec.rb
-  type: WeightedRspecGrader
-assign-1-part-4:
-  uri: http://s3.amazonaws.com/saasbook/assign1/part4_spec.rb
-  type: WeightedRspecGrader
-assign-1-part-5:
-  uri: http://s3.amazonaws.com/saasbook/assign1/part5_spec.rb
-  type: WeightedRspecGrader
+# assign-1-part-1:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part1_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-2:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part2_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-3:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part3_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-4:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part4_spec.rb
+#   type: WeightedRspecGrader
+# assign-1-part-5:
+#   uri: http://s3.amazonaws.com/saasbook/assign1/part5_spec.rb
+#   type: WeightedRspecGrader
+# 
+# assign-2-part-1:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part1_spec.rb
+#   type: WeightedRspecGrader
+# assign-2-part-2:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part2_spec.rb
+#   type: WeightedRspecGrader
+# assign-2-part-3:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part3_spec.rb
+#   type: HerokuRspecGrader
+# assign-2-part-4:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part4_spec.rb
+#   type: HerokuRspecGrader
+# assign-2-part-5:
+#   uri: http://s3.amazonaws.com/saasbook/assign2/part5_spec.rb
+#   type: HerokuRspecGrader
 
-assign-2-part-1:
-  uri: http://s3.amazonaws.com/saasbook/assign2/part1_spec.rb
-  type: WeightedRspecGrader
-assign-2-part-2:
-  uri: http://s3.amazonaws.com/saasbook/assign2/part2_spec.rb
-  type: WeightedRspecGrader
-assign-2-part-3:
-  uri: http://s3.amazonaws.com/saasbook/assign2/part3_spec.rb
-  type: HerokuRspecGrader
-assign-2-part-4:
-  uri: http://s3.amazonaws.com/saasbook/assign2/part4_spec.rb
-  type: HerokuRspecGrader
-assign-2-part-5:
-  uri: http://s3.amazonaws.com/saasbook/assign2/part5_spec.rb
-  type: HerokuRspecGrader
+assign-3-part-1:
+  uri: hw3.yml
+  type: HW3Grader

hw3.yml

@@ -21,14 +21,14 @@ features:
       - &step2
         FEATURE: features/filter_movie_list.feature
         version: 1
-        desc: "results -= Amelie"
+        desc: "results -= R movies"
         failures:
           - *restrict_pg_r
           - *all_x_selected
       - &step3
         FEATURE: features/filter_movie_list.feature
         version: 2
-        desc: "results -= Raiders of the Lost Ark"
+        desc: "results -= PG movies"
         failures:
           - *restrict_pg_r
           - *all_x_selected
@@ -51,13 +51,13 @@ features:
       - &step6
         FEATURE: features/filter_movie_list.feature
         version: 5
-        desc: "results += Aladdin"
+        desc: "results += G movies"
         failures:
           - *restrict_pg_r
       - &step7
         FEATURE: features/filter_movie_list.feature
         version: 6
-        desc: "results += Chocolat"
+        desc: "results += PG-13 movies"
         failures:
           - *restrict_pg_r
 #======= Same as step6
@@ -70,7 +70,7 @@ features:
       - &step9
         FEATURE: features/filter_movie_list.feature
         version: 8
-        desc: "results += The Help"
+        desc: "results += PG-13 movies"
         failures:
           - *restrict_pg_r
 #======= Same as step2
@@ -84,13 +84,13 @@ features:
       - &step11
         FEATURE: features/filter_movie_list.feature
         version: 10
-        desc: "results -= Chicken Run"
+        desc: "results -= G movies"
         failures:
           - *all_x_selected
       - &step12
         FEATURE: features/filter_movie_list.feature
         version: 11
-        desc: "results -= Chocolat"
+        desc: "results -= PG-13 movies"
         failures:
           - *all_x_selected
       - &step13
@@ -221,8 +221,8 @@ features:
   - &step34
     FEATURE: features/sort_movie_list.feature
     version: 28
-    desc: "results = [The Help]"
-  - &step35
-    FEATURE: features/sort_movie_list.feature
-    version: 29
-    desc: "results = [Chocolat]"
+    desc: "results = [PG-13 movies]"
+#  - &step35
+#    FEATURE: features/sort_movie_list.feature
+#    version: 29
+#    desc: "results = [PG-13 movies]"

lib/auto_grader_subprocess.rb

@@ -29,7 +29,7 @@ def self.run_autograder_subprocess(submission, spec, grader_type)
         }
       when 'HW3Grader'
         {
-          :timeout => 300,
+          :timeout => 400,
           :cmd => %Q{./grade3 -a ../rottenpotatoes "#{file.path}" "#{spec}"}
         }
       else

lib/graders/feature_grader.rb

@@ -64,6 +64,8 @@ def initialize(features_archive, grading_rules={})
     end
 
     $config = {:mt => grading_rules.has_key?(:mt) ? grading_rules[:mt] : true} # TODO merge all the configs
+    $config[:mt] = (ENV["AG_MT"] =~ /1|true/i) if ENV.has_key?("AG_MT")
+    $config[:mt] = false
 
     @temp = TempArchiveFile.new(@features_archive)
     @logpath = File.expand_path(File.join('.', 'log', "hw3_#{File.basename @temp.path}.log"))
@@ -92,9 +94,11 @@ def grade!
       start_time = Time.now
 
       score = Feature.total(@features)   # TODO integrate Score
+
       @raw_score, @raw_max = score.points, score.max
 
-      # log "Completed in #{Time.now-start_time} seconds."
+      log "Total score: #{@raw_score} / #{@raw_max}"
+      log "Completed in #{Time.now-start_time} seconds."
       dump_output
     ensure
       @temp.destroy if @temp

lib/graders/feature_grader/cuke_runner

@@ -4,20 +4,38 @@ def main
   require 'rubygems'
   require 'cucumber'
   fpath = File.dirname(ENV["FEATURE"])
-  config = Cucumber::Cli::Configuration.new
-  config.parse! [
-    '--profile=default',
+
+  opts = [
+    "--no-color",
+    "--no-profile",
+    "--backtrace",
     "--require=#{File.join(fpath, 'support/')}",
     "--require=#{File.join(fpath, 'step_definitions/')}",
+    "--require=#{File.join('features', 'support', 'env.rb')}",  # the bootstrapped one
+    #"--require=#{File.join('features', 'support', '_rag.rb')}",
+    "TEST_DB=#{ENV['TEST_DB']}",
+    "version=#{ENV['version']}",   # XXX envvars have to be explicitly passed to drb...
     ENV["FEATURE"]
   ]
-  runner = Cucumber::Runtime.new(config)
-  runner.run!
+
+  opts.unshift "--drb" if ENV.has_key?("DRB") ? (ENV["DRB"] =~ /1|true/i) : true
+  opts.unshift "--port=#{ENV["DRB_PORT"]}" if ENV.has_key?("DRB_PORT")
+
+#  config = Cucumber::Cli::Configuration.new
+#  config.parse! opts
+#  runner = Cucumber::Runtime.new(config)
+#  runner.run!
+
+  #out, err = File.open("out_#{File.basename ENV['TEST_DB']}",'w+'), File.open("err_#{File.basename ENV['TEST_DB']}",'w+')
+#  m = Cucumber::Cli::Main.new(opts, out, err)
+#  m.execute!
+
+  Cucumber::Cli::Main.execute(opts)
 end
 
 begin
   main
 rescue Object => e
   puts "*** ERROR: #{e.inspect}"
-  puts e.backtrace
+  #puts e.backtrace
 end

lib/graders/feature_grader/feature.rb

@@ -123,7 +123,6 @@ def run!
 
         raise(TestFailedError, "Failed to find test db") unless File.readable? SOURCE_DB
         FileUtils.cp SOURCE_DB, h["TEST_DB"]
-#        Open3.popen3(h, "bundle exec rake cucumber", popen_opts) do |stdin, stdout, stderr, wait_thr|
         Open3.popen3(h, $CUKE_RUNNER, popen_opts) do |stdin, stdout, stderr, wait_thr|
           exit_status = wait_thr.value
 
@@ -133,7 +132,7 @@ def run!
         end
 
       rescue => e
-        log "test failed: #{e.inspect}".red.bold
+        log "test failed: #{e.inspect}"#.red.bold
         log e.backtrace
         raise TestFailedError, "test failed to run b/c #{e.inspect}"
 
@@ -144,19 +143,23 @@ def run!
 
       if self.correct?
         #log "Test #{h.inspect} passed.".green
-        log "Test passed."
-        score.pass
+        if self.desc           # XXX hack to indicate a mod
+          score.pass
+        else
+          score += 26.0/8.0    # XXX give happy paths are worth 20% of total
+        end
+        log "Test passed. (+#{score.max})"
         score += Feature.total(@if_pass)
       else
         #log "Test #{h.inspect} failed".red
-        log "Test failed"
         begin
           self.correct!
         rescue TestFailedError, IncorrectAnswer => e
           log e.message
         end
         log lines.collect {|l| "| #{l}"}
         score.fail
+        log "Test failed. (-#{score.max})"
       end
 
       return score

lib/graders/feature_grader/score.rb

@@ -10,7 +10,7 @@ def +(other)
     case other
     when Score
       Score.new(@points+other.points, @max+other.max)
-    when Integer
+    when Numeric
       Score.new(@points+other, @max+other)
     else
       raise ArgumentError