Get DataShop log conversion working on UNIX, plus improvements.

Add perl one-liners to DataShop conversion instructions
Add number of problems solved for each student to onelog.pl
Fix parsing of header lines with new format dates.

LogProcessing/OLI Log Processing.html

@@ -185,13 +185,15 @@ <h4>Concatenated log format</h4>
 text file in which the lines end with carriage-return linefeed pairs.
 One effect of this is that when viewed in a text editor, there will
 appears to be a blank line after each log file. <br>
+
 <h2><span style="font-weight: bold;">Anonymizing the logs</span></h2>
 <h2><span style="font-weight: bold;"></span></h2>
 <br>
 There are different scripts for anonymizing USNA logs, where our
 students are supposed to use their alphas, aka mid numbers, as user
 ids, and anonymizing arbitrary student logs, where the ids may take any
 form.<br>
+
 <h5>Anonymizing USNA midshipmen logs: mkanon-usna.pl<br>
 </h5>
 To anonymize mid logs, obtain the file mkanon-usna.pl.template. Edit it
@@ -217,6 +219,13 @@ <h5>Anonymizing USNA midshipmen logs: mkanon-usna.pl<br>
 instructor or TA accounts. So, these untranslated ids should then be
 identifiable in the anonymized output by the fact that they differ in
 form from all the student ids. <br>
+
+<p>The user id appears twice in each session:  in cases where there
+is some ambiguity, it can be useful to compare the two:
+<pre>
+cat log/Spring2009-USNA.dat | perl -ne 'm/# Log .* by (.+) on / and print "$1 "; m/read-student-info "(.*)"/ and print "$1\n"' | sort | uniq
+</pre>
+
 <h5>Using OLI class rosters</h5>
 It is generally useful to save copies of the OLI roster tables. OLI
 does not provide any method to export the rosters, so the method for
@@ -396,7 +405,8 @@ <h2>Converting Raw Logs to DataShop XML Format</h2>
 conversion, this set of directories should be zipped up and delivered
 to the DataShop for
 dropoff. The DataShop should give you ssh access to a dropbox location
-on their "cooker" server for this purpose.<br>
+on <code>the-cooker.pslc.cs.cmu.edu</code> for this purpose.<br>
+
 <h5>Info files</h5>
 Although the basic conversion is simple, the converter makes use of
 several external files to patch in information not included in the
@@ -446,6 +456,12 @@ <h6>classmap.txt: student to class id mapping<br>
 <br>
 &nbsp;&nbsp;&nbsp; perl mapidcol.pl idmap.txt 1 &lt; raw-classmap.txt
 &gt; classmap.txt<br>
+<br>
+Or use the one-liner:
+<pre>
+perl -p -e "s/(\w+) +(\w+)/\2 USNA/" log/Spring2009-USNA-mapping.txt > Info/classmap.txt
+</pre>
+
 <h6>Class information</h6>
 The Info file class-XXX.xml specifies the full xml element to patch
 into the conversion output for specifying the class of students mapped

LogProcessing/log2xml.pl

@@ -120,7 +120,7 @@
 
 # load the problem to unit mapping table, if it exists
 if (open UNITMAP, "<Info/unitmap.txt") {
-     %unitmap = map /(.*)\t([^\r]*)/, <UNITMAP>;
+     %unitmap = map /(.*)\t([^\r^\n]*)/, <UNITMAP>;
      close UNITMAP;
      $have_unitmap = 1;
      #print STDERR "loaded unit map\n";
@@ -130,15 +130,15 @@
 # we may get a condition from a conditionmap file, or else from set-condition
 # above. If both are set, condition map will override.
 if (open CONDITIONMAP, "<Info/conditionmap.txt") {
-	%conditionmap = map /(.*)\t([^\r]*)/, <CONDITIONMAP>;
+	%conditionmap = map /(.*)\t([^\r^\n]*)/, <CONDITIONMAP>;
         close CONDITIONS;
 	$have_conditionmap = 1;
      #print STDERR "loaded condition map\n";
 }
 
 # load the student to class mapping table, if it exists
 if (open CLASSMAP, "<Info/classmap.txt") {
-	%classmap = map /(.*)\t([^\r]*)/, <CLASSMAP>;
+	%classmap = map /(.*) +([^\r^\n]+)/, <CLASSMAP>;
         close CLASSMAP;
 	$have_classmap = 1;
 }
@@ -153,10 +153,15 @@
     # Log header line begins a new log. We could reset and get date from this, but
     # in fact we get it from initial set-session-id call below. However, need to 
     # get the year from here since it's not in the session id.
-    if (/# Log of Andes session begun [\w]+, [\w]+ [\d]+, ([\d]+) [\d:]+ by/) {
+    if (/# Log of Andes session begun \w+, \w+ \d+, (\d+) /) {
         # Example:                   Monday, October 22, 2005 02:13:52
     	$year = $1;
     }
+    # case with AM/PM time zone format
+    elsif (/# Log of Andes session begun [\w]+ \d+ \w+ (\d+) /) {
+        # Example:                   Sun 08 Feb 2009 01:49 PM EST
+    	$year = $1;
+    }
 
     next if /^#/;  # skip log header lines 
 

LogProcessing/mkanon-usna.pl.template

@@ -18,7 +18,7 @@
 #            "blindmelon" => "m094098");
 
 if (open MERGEFILE, "<merge-ids.txt") {
-     %merge = map /(.*)\t([^\r]*)/, <MERGEFILE>;
+     %merge = map /(.*)\t(\w*)/, <MERGEFILE>;
      close MERGEFILE;
 
      #print STDERR "loaded merge map:\n";

LogProcessing/mkanon.pl

@@ -25,7 +25,7 @@
 
 # load existing mapping from specified mapping file
 open MAPFILE, "<$filename" or die "Couldn't open $filename for reading: $!\n"  ;
-%idmap = map /(.*)\t(.*)\r/, <MAPFILE>; # gulps in hash. Assumes DOS mode \r is read
+%idmap = map /(.*)\t(\w*)/, <MAPFILE>; # gulps in hash. 
 close MAPFILE;
 if (0) { # debugging printout
 	$maplength = (keys idmap);
@@ -124,7 +124,7 @@
 if ($modified_map) {
  open MAPFILE, ">$filename" or die "Couldn't open $filename for updating: $!\n";
  while( ($k, $v) = each %idmap) {
-        print MAPFILE "$k\t$v\r\n";
+        print MAPFILE "$k\t$v\n";
  }
  close MAPFILE;
 }

LogProcessing/onelog.pl

@@ -367,6 +367,15 @@ sub timestamp {
 #############################################################################
 
 
+# print out number of problems attempted by each student
+# This is not affected by any of the cutoffs
+if (1) {
+    foreach $student (keys %times) {
+	my $i =  keys %{$times{$student}};
+	print "$student $i\n";
+    }
+}    
+
 # print out score histogram in Mathematica notation
 # This is not affected by any of the cutoffs
 if (0) {