Add skipping logic

tools/submission/submission_checker.py

@@ -608,6 +608,11 @@ def get_args():
         action="store_true",
         help="skips the check of extra files inside the root submission dir",
     )
+    parser.add_argument(
+        "--scenarios-to-skip",
+        help="Delimited list input of scenarios to skip. i.e. if you only have Offline results, pass in 'Server'",
+        type=str
+    )
     args = parser.parse_args()
     return args
 
@@ -1234,6 +1239,7 @@ def check_results_dir(
     skip_empty_files_check=False,
     skip_check_power_measure_files=False,
     skip_extra_files_in_root_check=False,
+    scenarios_to_skip=[]
 ):
     """
     Walk the results directory and do the checking.
@@ -1634,6 +1640,10 @@ def log_result(
                         # some submissions in v0.5 use lower case scenarios - map them for now
                         scenario_fixed = SCENARIO_MAPPING.get(scenario, scenario)
 
+                        # Skip scenario for debug purposes
+                        if scenario in scenarios_to_skip:
+                            continue
+
                         # we are looking at ./$division/$submitter/results/$system_desc/$model/$scenario,
                         #   ie ./closed/mlperf_org/results/t4-ort/bert/Offline
                         name = os.path.join(
@@ -1715,7 +1725,7 @@ def log_result(
                             )
                             errors += 1
                             continue
-                        else:
+                        elif scenario not in scenarios_to_skip:
                             diff = files_diff(list_files(acc_path), REQUIRED_ACC_FILES)
                             if diff:
                                 log.error(
@@ -1879,6 +1889,8 @@ def log_result(
                         # check if compliance dir is good for CLOSED division
                         compliance = 0 if is_closed_or_network else 1
                         if is_closed_or_network and not skip_compliance:
+                            if scenario in scenarios_to_skip:
+                                continue
                             compliance_dir = os.path.join(
                                 division,
                                 submitter,
@@ -1929,6 +1941,10 @@ def log_result(
                                 results[name] = None
                                 log.error("%s is OK but accuracy has issues", name)
 
+                    # Discard scenarios that we want to skip
+                    for scenario in scenarios_to_skip:
+                        required_scenarios.discard(scenario)
+
                     if required_scenarios:
                         name = os.path.join(results_path, system_desc, model_name)
                         if is_closed_or_network:
@@ -2393,6 +2409,13 @@ def main():
         skip_power_check=args.skip_power_check,
     )
 
+    if args.scenarios_to_skip:
+        scenarios_to_skip = [
+            scenario for scenario in args.scenarios_to_skip.split(',')
+        ]
+    else:
+        scenarios_to_skip = []
+
     with open(args.csv, "w") as csv:
         os.chdir(args.input)
         # check results directory
@@ -2405,7 +2428,8 @@ def main():
             args.skip_meaningful_fields_emptiness_check,
             args.skip_empty_files_check,
             args.skip_check_power_measure_files,
-            args.skip_extra_files_in_root_check
+            args.skip_extra_files_in_root_check,
+            scenarios_to_skip
         )
 
     # log results

tools/submission/truncate_accuracy_log.py

@@ -46,6 +46,12 @@ def get_args():
     parser.add_argument("--output", help="new submission directory")
     parser.add_argument("--submitter", required=True, help="filter to submitter")
     parser.add_argument("--backup", help="directory to store the original accuacy log")
+    parser.add_argument(
+        "--scenarios-to-skip",
+        help="Delimited list input of scenarios to skip. i.e. if you only have Offline results, pass in 'Server'",
+        type=str
+    )
+
 
     args = parser.parse_args()
     if not args.output and not args.backup:
@@ -104,7 +110,7 @@ def copy_submission_dir(src, dst, filter_submitter):
                             os.path.join(dst, division, submitter))
 
 
-def truncate_results_dir(filter_submitter, backup):
+def truncate_results_dir(filter_submitter, backup, scenarios_to_skip):
     """Walk result dir and 
        write a hash of mlperf_log_accuracy.json to accuracy.txt
        copy mlperf_log_accuracy.json to a backup location
@@ -131,6 +137,8 @@ def truncate_results_dir(filter_submitter, backup):
                 for system_desc in list_dir(log_path):
                     for model in list_dir(log_path, system_desc):
                         for scenario in list_dir(log_path, system_desc, model):
+                            if scenario in scenarios_to_skip:
+                                continue
                             for test in list_dir(log_path, system_desc, model, scenario):
 
                                 name = os.path.join(log_path, system_desc, model, scenario)
@@ -200,8 +208,16 @@ def main():
         src_dir = args.output
 
     os.chdir(src_dir)
+
+    if args.scenarios_to_skip:
+        scenarios_to_skip = [
+            scenario for scenario in args.scenarios_to_skip.split(',')
+        ]
+    else:
+        scenarios_to_skip = []
+
     # truncate results directory
-    truncate_results_dir(args.submitter, args.backup)
+    truncate_results_dir(args.submitter, args.backup, scenarios_to_skip)
 
     backup_location = args.output or args.backup
     log.info("Make sure you keep a backup of %s in case mlperf wants to see the original accuracy logs", backup_location)