Address review comments, small reorganization

server/jobs/moss.py

@@ -3,7 +3,6 @@
 import shlex
 import glob
 import tempfile
-import ntpath
 import re
 
 from server.models import Assignment, Backup, db
@@ -21,14 +20,13 @@ def submit_to_moss(moss_id=None, file_regex="*", assignment_id=None, language=No
         return
 
     subms = assign.course_submissions(include_empty=False)
-    subm_keys = set(s['backup']['id'] for s in subms)
 
-    seen = set()
+    subm_keys = set()
     for subm in subms:
-        if subm['backup']['id'] in seen:
+        if subm['backup']['id'] in subm_keys:
             continue
         else:
-            seen.add(subm['backup']['id'])
+            subm_keys.add(subm['backup']['id'])
 
         if subm['group']:
             logger.info("{} -> {}".format(encode_id(subm['backup']['id']),
@@ -50,9 +48,13 @@ def submit_to_moss(moss_id=None, file_regex="*", assignment_id=None, language=No
         # Copy in the moss script
         with open('server/jobs/moss-submission.pl', 'r') as f:
             moss_script = f.read()
-            moss_script = moss_script.replace('YOUR_USER_ID_HERE', str(moss_id))
-            with open(tmp_dir + "/moss.pl", 'w') as script:
-                script.write(moss_script)
+
+        moss_script = moss_script.replace('YOUR_USER_ID_HERE', str(moss_id))
+        with open(tmp_dir + "/moss.pl", 'w') as script:
+            script.write(moss_script)
+
+        match_pattern = re.compile(file_regex)
+        ignored_files = set()
 
         for backup in backup_query:
             # Write file into file
@@ -69,11 +71,25 @@ def submit_to_moss(moss_id=None, file_regex="*", assignment_id=None, language=No
             for file in contents:
                 if file == 'submit':  # ignore fake file from ok-client
                     continue
-                with open(dest_dir + file, 'w') as f:
-                    f.write(contents[file])
+
+                if match_pattern.match(file):
+                    with open(dest_dir + file, 'w') as f:
+                        f.write(contents[file])
+                else:
+                    ignored_files.add(file)
+
+        # tmp_dir contains folders of the form: backup_hashid/file1.py
+        os.chdir(tmp_dir)
+        all_student_files = glob.glob("*/*")
 
         logger.info("Wrote all files to {}".format(tmp_dir))
 
+        if ignored_files:
+            logger.info("Regex {} ignored files with names: {}".format(file_regex,
+                                                                       ignored_files))
+        else:
+            logger.info("Regex {} has captured all possible files".format(file_regex))
+
         template_files = []
         for template in assign.files:
             dest = "{}/{}".format(tmp_dir, template)
@@ -86,30 +102,13 @@ def submit_to_moss(moss_id=None, file_regex="*", assignment_id=None, language=No
 
         templates = ' '.join(["-b {file}".format(file=f) for f in template_files])
 
-        os.chdir(tmp_dir)
-        all_student_files = glob.glob("*/*")
-        chosen_files, available_file_names = [], []
-        match_pattern = re.compile(file_regex)
-
-        logger.info("Running with regex pattern: {}".format(file_regex))
-
-        for file in all_student_files:
-            fname = ntpath.basename(file)
-            available_file_names.append(fname)
-            if match_pattern.match(fname):
-                chosen_files.append(file)
-
-        if not chosen_files:
-            logger.info(("Regex: {} matched none of {} possible choices"
-                         .format(file_regex, len(all_student_files))))
-            if all_student_files:
-                logger.info("Sample file paths: {}".format(available_file_names[0:5]))
+        if not all_student_files:
             raise Exception("Did not match any files")
 
         # Ensure that all of the files are in the tmp_dir (and not elsewhere)
         command = ("perl moss.pl -l {lang} {templates} -d {folder}"
                    .format(lang=language, templates=templates,
-                           folder=' '.join(chosen_files)))
+                           folder=' '.join(all_student_files)))
 
         logger.info("Running {}".format(command[:100] + ' ...'))