Cdf Json Fixes

src/main/java/network/brightspots/rcv/ContestConfig.java

@@ -76,7 +76,6 @@ class ContestConfig {
   private static final int MAX_MULTI_SEAT_BOTTOMS_UP_PERCENTAGE_THRESHOLD = 100;
   private static final long MIN_RANDOM_SEED = -140737488355328L;
   private static final long MAX_RANDOM_SEED = 140737488355327L;
-  private static final String JSON_EXTENSION = ".json";
   private static final String MAX_SKIPPED_RANKS_ALLOWED_UNLIMITED_OPTION = "unlimited";
   private static final String MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION = "max";
   static final String SUGGESTED_MAX_RANKINGS_ALLOWED = MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION;
@@ -100,9 +99,7 @@ private ContestConfig(RawContestConfig rawConfig, String sourceDirectory) {
   }
 
   static boolean isCdf(CvrSource source) {
-    return getProvider(source) == Provider.CDF
-        && source.getFilePath() != null
-        && source.getFilePath().toLowerCase().endsWith(JSON_EXTENSION);
+    return getProvider(source) == Provider.CDF;
   }
 
   static ContestConfig loadContestConfig(RawContestConfig rawConfig, String sourceDirectory) {
@@ -209,6 +206,15 @@ static boolean passesBasicCvrSourceValidation(CvrSource source) {
           Logger.log(Level.SEVERE, "overvoteDelimiter is invalid.");
         }
       } else {
+        if (provider == Provider.CDF) {
+          if (!source.getFilePath().toLowerCase().endsWith(".xml") && !source.getFilePath()
+              .toLowerCase().endsWith(".json")) {
+            Logger
+                .severe("CDF source files must be .json or .xml! Unexpected file extension for: %s",
+                    source.getFilePath());
+            sourceValid = false;
+          }
+        }
         if (fieldIsDefinedButShouldNotBeForProvider(
             source.getFirstVoteColumnIndex(),
             "firstVoteColumnIndex",
@@ -250,17 +256,19 @@ static boolean passesBasicCvrSourceValidation(CvrSource source) {
         }
       }
 
-      if (isNullOrBlank(source.getContestId()) &&
-          (provider == Provider.DOMINION || provider == Provider.HART
-              || provider == Provider.CLEAR_BALLOT)) {
+      boolean providerRequiresContestId = provider == Provider.DOMINION ||
+          provider == Provider.HART ||
+          provider == Provider.CLEAR_BALLOT ||
+          provider == Provider.CDF;
+
+      if (isNullOrBlank(source.getContestId()) && providerRequiresContestId) {
         sourceValid = false;
         Logger.log(
             Level.SEVERE,
             String.format("contestId must be defined for CVR source with provider \"%s\"!",
                 getProvider(source).toString()));
       } else if (
-          !(provider == Provider.DOMINION || provider == Provider.HART
-              || provider == Provider.CLEAR_BALLOT) &&
+          !(providerRequiresContestId) &&
               fieldIsDefinedButShouldNotBeForProvider(
                   source.getContestId(),
                   "contestId",
@@ -1025,22 +1033,11 @@ void setCandidateExclusionStatus(String candidateCode, boolean excluded) {
   }
 
   // perform pre-processing on candidates:
-  // 1) if there are any CDF input sources extract candidates names from them
-  // 2) build map of candidate ID to candidate name
-  // 3) generate tie-break ordering if needed
+  // 1) build map of candidate ID to candidate name
+  // 2) generate tie-break ordering if needed
+  // 3) add uwi candidate if needed
   private void processCandidateData() {
     candidateCodeToNameMap = new HashMap<>();
-
-    for (RawContestConfig.CvrSource source : rawConfig.cvrFileSources) {
-      // for any CDF sources extract candidate names
-      if (isCdf(source)) {
-        String cvrPath = resolveConfigPath(source.getFilePath());
-        CommonDataFormatReader reader = new CommonDataFormatReader(cvrPath, this);
-        candidateCodeToNameMap = reader.getCandidates();
-        candidatePermutation.addAll(candidateCodeToNameMap.keySet());
-      }
-    }
-
     if (rawConfig.candidates != null) {
       for (RawContestConfig.Candidate candidate : rawConfig.candidates) {
         String code = candidate.getCode();

src/main/java/network/brightspots/rcv/GuiConfigController.java

@@ -431,7 +431,7 @@ public void buttonCvrFilePathClicked() {
       case CDF -> {
         FileChooser fc = new FileChooser();
         fc.setInitialDirectory(new File(FileUtils.getUserDirectory()));
-        fc.getExtensionFilters().add(new ExtensionFilter("JSON files", "*.json"));
+        fc.getExtensionFilters().add(new ExtensionFilter("JSON and XML files", "*.json", "*.xml"));
         fc.setTitle("Select " + provider + " Cast Vote Record File");
         openFile = fc.showOpenDialog(GuiContext.getInstance().getMainWindow());
       }
@@ -902,12 +902,7 @@ public LocalDate fromString(String string) {
           textFieldCvrPrecinctCol.setDisable(false);
           textFieldCvrOvervoteDelimiter.setDisable(false);
         }
-        case CDF -> {
-          buttonAddCvrFile.setDisable(false);
-          textFieldCvrFilePath.setDisable(false);
-          buttonCvrFilePath.setDisable(false);
-        }
-        case CLEAR_BALLOT, DOMINION, HART -> {
+        case CLEAR_BALLOT, DOMINION, HART, CDF -> {
           buttonAddCvrFile.setDisable(false);
           textFieldCvrFilePath.setDisable(false);
           buttonCvrFilePath.setDisable(false);

src/main/java/network/brightspots/rcv/Logger.java

@@ -122,6 +122,18 @@ static void log(Level level, String format, Object... obj) {
     logger.log(level, String.format(format, obj));
   }
 
+  static void info(String format, Object... obj) {
+    logger.log(Level.INFO, String.format(format, obj));
+  }
+
+  static void warning(String format, Object... obj) {
+    logger.log(Level.WARNING, String.format(format, obj));
+  }
+
+  static void severe(String format, Object... obj) {
+    logger.log(Level.SEVERE, String.format(format, obj));
+  }
+
   // add logging to the provided text area for display to user in the GUI
   static void addGuiLogging(TextArea textArea) {
     java.util.logging.Handler guiHandler =

src/main/java/network/brightspots/rcv/ResultsWriter.java

@@ -65,7 +65,8 @@ class ResultsWriter {
   private static final String CDF_GPU_ID_FORMAT = "gpu-%d";
   private static final String CDF_REPORTING_DEVICE_ID = "rd-001";
 
-  private static final Map<String, String> candidateCodeToCdfId = new HashMap<>();
+  private static final Map<String, String> candidateCodeToContestSelectionId = new HashMap<>();
+  private static final Map<String, String> candidateCodeToCandidateId = new HashMap<>();
 
   // number of rounds needed to elect winner(s)
   private int numRounds;
@@ -136,18 +137,27 @@ private static String generateCvrSnapshotId(String cvrId, Integer round) {
         : String.format("ballot-%s", cvrId);
   }
 
-  private static String getCdfIdForCandidateCode(String code) {
-    String id = candidateCodeToCdfId.get(code);
+  // generates an internal ContestSelectionId based on a candidate code
+  private static String getContestSelectionIdForCandidateCode(String code) {
+    String id = candidateCodeToContestSelectionId.get(code);
     if (id == null) {
-      id =
-          code.startsWith("cs-")
-              ? code
-              : String.format("cs-%s", sanitizeStringForOutput(code).toLowerCase());
-      candidateCodeToCdfId.put(code, id);
+      id = String.format("cs-%s", sanitizeStringForOutput(code).toLowerCase());
+      candidateCodeToContestSelectionId.put(code, id);
     }
     return id;
   }
 
+  // generates an internal CandidateId based on a candidate code
+  private static String getCandidateIdForCandidateCode(String code) {
+    String id = candidateCodeToCandidateId.get(code);
+    if (id == null) {
+      id = String.format("c-%s", sanitizeStringForOutput(code).toLowerCase());
+      candidateCodeToCandidateId.put(code, id);
+    }
+    return id;
+  }
+
+
   // Instead of a map from rank to list of candidates, we need a sorted list of candidates
   // with the ranks they were given. (Ordinarily a candidate will have only a single rank, but they
   // could have multiple ranks if the ballot duplicates the candidate, i.e. assigns them multiple
@@ -751,7 +761,7 @@ private Map<String, Object> generateCvrSnapshotMap(
 
       selectionMapList.add(
           Map.ofEntries(
-              entry("ContestSelectionId", getCdfIdForCandidateCode(candidateCode)),
+              entry("ContestSelectionId", getContestSelectionIdForCandidateCode(candidateCode)),
               entry("SelectionPosition", selectionPositionMapList),
               entry("@type", "CVR.CVRContestSelection")));
     }
@@ -772,31 +782,36 @@ private Map<String, Object> generateCvrSnapshotMap(
   private Map<String, Object> generateCdfMapForElection() {
     HashMap<String, Object> electionMap = new HashMap<>();
 
+    // containers for election-level data
     List<Map<String, Object>> contestSelections = new LinkedList<>();
+    List<Map<String, Object>> candidates = new LinkedList<>();
+
+    // iterate all candidates and create Candidate and ContestSelection objects for them
     List<String> candidateCodes = new LinkedList<>(config.getCandidateCodeList());
     Collections.sort(candidateCodes);
     for (String candidateCode : candidateCodes) {
-      Map<String, String> codeMap =
+      candidates.add(
           Map.ofEntries(
-              entry("@type", "CVR.Code"),
-              entry("Type", "other"),
-              entry("OtherType", "vendor-label"),
-              entry("Value", config.getNameForCandidateCode(candidateCode)));
+              entry("@id", getCandidateIdForCandidateCode(candidateCode)),
+              entry("Name", candidateCode)));
 
       contestSelections.add(
           Map.ofEntries(
-              entry("@id", getCdfIdForCandidateCode(candidateCode)),
+              entry("@id", getContestSelectionIdForCandidateCode(candidateCode)),
               entry("@type", "CVR.ContestSelection"),
-              entry("Code", new Map[]{codeMap})));
+              entry("CandidateIds", new String[]{getCandidateIdForCandidateCode(candidateCode)})));
     }
 
     Map<String, Object> contestJson =
         Map.ofEntries(
             entry("@id", CDF_CONTEST_ID),
+            entry("@type", "CVR.CandidateContest"),
             entry("ContestSelection", contestSelections),
-            entry("@type", "CVR.CandidateContest"));
+            entry("Name", config.getContestName())
+        );
 
     electionMap.put("@id", CDF_ELECTION_ID);
+    electionMap.put("Candidate", candidates);
     electionMap.put("Contest", new Map[]{contestJson});
     electionMap.put("ElectionScopeId", CDF_GPU_ID);
     electionMap.put("@type", "CVR.Election");

src/main/java/network/brightspots/rcv/TabulatorSession.java

@@ -258,7 +258,8 @@ private List<CastVoteRecord> parseCastVoteRecords(ContestConfig config, Set<Stri
       try {
         if (ContestConfig.isCdf(source)) {
           Logger.log(Level.INFO, "Reading CDF cast vote record file: %s...", cvrPath);
-          new CommonDataFormatReader(cvrPath, config).parseCvrFile(castVoteRecords);
+          new CommonDataFormatReader(cvrPath, config, source.getContestId())
+              .parseCvrFile(castVoteRecords);
           continue;
         } else if (ContestConfig.getProvider(source) == Provider.CLEAR_BALLOT) {
           Logger
@@ -335,8 +336,11 @@ private List<CastVoteRecord> parseCastVoteRecords(ContestConfig config, Set<Stri
         encounteredSourceProblem = true;
       } catch (CvrParseException e) {
         encounteredSourceProblem = true;
+      } catch (Exception exception) {
+        Logger.severe("Unexpected error parsing source file %s", cvrPath);
+        encounteredSourceProblem = true;
       }
-    }
+  }
 
     if (encounteredSourceProblem) {
       Logger.log(Level.SEVERE, "Parsing cast vote records failed!");

src/test/java/network/brightspots/rcv/TabulatorTests.java

@@ -26,6 +26,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -117,9 +118,10 @@ private static String getTestFilePath(String stem, String suffix) {
   // helper function to support running various tabulation tests
   private static void runTabulationTest(String stem) {
     String configPath = getTestFilePath(stem, "_config.json");
+    Logger.info("Running tabulation test: %s\nTabulating config file: %s...", stem, configPath);
     TabulatorSession session = new TabulatorSession(configPath);
     session.tabulate();
-
+    Logger.info("Examining tabulation test results...");
     String timestampString = session.getTimestampString();
     ContestConfig config = ContestConfig.loadContestConfig(configPath);
     assertNotNull(config);
@@ -140,7 +142,6 @@ private static void runTabulationTest(String stem) {
           "_contest_" + config.rawConfig.cvrFileSources.get(0).getContestId() + "_expected.csv");
       assertTrue(fileCompare(session.getConvertedFilesWritten().get(0), expectedPath));
     }
-
     // test passed so cleanup test output folder
     File outputFolder = new File(session.getOutputPath());
     if (outputFolder.listFiles() != null) {
@@ -156,6 +157,7 @@ private static void runTabulationTest(String stem) {
         }
       }
     }
+    Logger.info("Test complete.");
   }
 
   private static void compareJsons(
@@ -183,14 +185,68 @@ private static void compareJson(
                 + "_expected_"
                 + jsonType
                 + ".json");
-    assertTrue(fileCompare(expectedPath, actualOutputPath));
+    Logger.info("Comparing files:\nGenerated: %s\nReference: %s", actualOutputPath, expectedPath);
+    if (fileCompare(expectedPath, actualOutputPath)) {
+      Logger.info("Files are equal.");
+    } else {
+      Logger.info("Files are different.");
+      fail();
+    }
   }
 
   @BeforeAll
   static void setup() {
     Logger.setup();
   }
 
+  @Test
+  @DisplayName("NIST XML CDF 2")
+  void nistXmlCdf2() {
+    runTabulationTest("nist_xml_cdf_2");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_city_tax_collector")
+  void unisynXmlCdfCityTaxCollector() {
+    runTabulationTest("unisyn_xml_cdf_city_tax_collector");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_city_mayor")
+  void unisynXmlCdfCityMayor() {
+    runTabulationTest("unisyn_xml_cdf_city_mayor");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_city_council_member")
+  void unisynXmlCdfCityCouncilMember() {
+    runTabulationTest("unisyn_xml_cdf_city_council_member");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_city_chief_of_police")
+  void unisynXmlCdfCityChiefOfPolice() {
+    runTabulationTest("unisyn_xml_cdf_city_chief_of_police");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_city_coroner")
+  void unisynXmlCdfCityCoroner() {
+    runTabulationTest("unisyn_xml_cdf_city_coroner");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_county_sheriff")
+  void unisynXmlCdfCountySheriff() {
+    runTabulationTest("unisyn_xml_cdf_county_sheriff");
+  }
+
+  @Test
+  @DisplayName("unisyn_xml_cdf_county_coroner")
+  void unisynXmlCdfCountyCoroner() {
+    runTabulationTest("unisyn_xml_cdf_county_coroner");
+  }
+
   @Test
   @DisplayName("Clear Ballot - Kansas Primary")
   void testClearBallotKansasPrimary() {