Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

direct Dominion tabulation #470

Merged
merged 5 commits into from
Jul 22, 2020
Merged

direct Dominion tabulation #470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6b6e82a
direct Dominion tabulation
tarheel Jul 20, 2020
8a3f366
enable specifying a Dominion CVR source path in the UI
tarheel Jul 20, 2020
04fa619
output generic CSV and add test coverage
tarheel Jul 22, 2020
97d6307
remove extra log line
tarheel Jul 22, 2020
440669b
filter out CVRs from other contests during the loading
tarheel Jul 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions src/main/java/network/brightspots/rcv/ContestConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ class ContestConfig {
private static final String MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION = "max";
static final String SUGGESTED_MAX_RANKINGS_ALLOWED = MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION;

static final String UNDECLARED_WRITE_INS = "Undeclared Write-ins";

static boolean isCdf(CvrSource source) {
return getProvider(source) == Provider.CDF
&& source.getFilePath() != null
Expand Down Expand Up @@ -498,16 +500,25 @@ private void validateCvrFileSources() {
}
}

if (isNullOrBlank(getContestId()) && getProvider(source) == Provider.HART) {
Provider provider = getProvider(source);

if (isNullOrBlank(getContestId()) &&
(provider == Provider.DOMINION || provider == Provider.HART)) {
isValid = false;
Logger.log(
Level.SEVERE,
"contestId is required for Hart files.");
} else if (!isNullOrBlank(getContestId()) && getProvider(source) != Provider.HART) {
String.format("contestId must be defined for CVR source with provider \"%s\"!",
getProvider(source).toString()));
} else if (
!(provider == Provider.DOMINION || provider == Provider.HART) &&
fieldIsDefinedButShouldNotBeForProvider(
getContestId(),
"contestId",
provider,
source.getFilePath())
) {
// helper will log error
isValid = false;
Logger.log(
Level.SEVERE,
"contestId may not be used with this type of CVR file.");
}
}
}
Expand Down Expand Up @@ -887,6 +898,7 @@ private Integer stringToIntWithOption(String rawInput, String optionFlag, Intege

enum Provider {
CDF("CDF"),
DOMINION("Dominion"),
ESS("ES&S"),
HART("Hart"),
PROVIDER_UNKNOWN("Provider unknown");
Expand Down Expand Up @@ -1065,7 +1077,7 @@ private void processCandidateData() {

String uwiLabel = getUndeclaredWriteInLabel();
if (!isNullOrBlank(uwiLabel)) {
candidateCodeToNameMap.put(uwiLabel, uwiLabel);
candidateCodeToNameMap.put(uwiLabel, UNDECLARED_WRITE_INS);
}
}
}
14 changes: 10 additions & 4 deletions src/main/java/network/brightspots/rcv/DominionCvrReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,9 @@ private static List<Candidate> getCandidates(String candidatePath) {
}

// parse Cvr json into CastVoteRecord objects and add them to the input list
void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
// (If contestId is specified, we'll only load CVRs for that contest.)
void readCastVoteRecords(List<CastVoteRecord> castVoteRecords, String contestId)
throws CvrParseException {
// read metadata files for precincts, precinct portions, contest, and candidates
Path precinctPath = Paths.get(manifestFolder, PRECINCT_MANIFEST);
this.precincts = getPrecinctData(precinctPath.toString());
Expand Down Expand Up @@ -149,16 +151,16 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseEx
}
// parse the cvr
Path cvrPath = Paths.get(manifestFolder, CVR_EXPORT);
parseCvrFile(cvrPath.toString(), castVoteRecords);
parseCvrFile(cvrPath.toString(), castVoteRecords, contestId);
if (castVoteRecords.isEmpty()) {
Logger.log(Level.SEVERE, "No cast vote record data found!");
throw new CvrParseException();
}
}

// parse the given file into a List of CastVoteRecords for tabulation
private void parseCvrFile(String filePath, List<CastVoteRecord> castVoteRecords) {

private void parseCvrFile(String filePath, List<CastVoteRecord> castVoteRecords,
String contestIdToLoad) {
// build a lookup map for candidates codes to optimize Cvr parsing
Map<String, Set<String>> contestIdToCandidateCodes = new HashMap<>();
for (Candidate candidate : this.candidates) {
Expand Down Expand Up @@ -234,6 +236,10 @@ private void parseCvrFile(String filePath, List<CastVoteRecord> castVoteRecords)
for (Object contestObject : contests) {
HashMap contest = (HashMap) contestObject;
String contestId = contest.get("Id").toString();
// skip this CVR if it's not for the contest we're interested in
if (contestIdToLoad != null && !contestId.equals(contestIdToLoad)) {
continue;
}
// validate contest id
if (!this.contests.containsKey(contestId)
|| !contestIdToCandidateCodes.containsKey(contestId)) {
Expand Down
27 changes: 13 additions & 14 deletions src/main/java/network/brightspots/rcv/GuiConfigController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,33 +416,32 @@ public void buttonClearDatePickerContestDateClicked() {
public void buttonCvrFilePathClicked() {
File openFile = null;

switch (getChoiceElse(choiceCvrProvider, Provider.PROVIDER_UNKNOWN)) {
case "CDF": {
String providerName = getChoiceElse(choiceCvrProvider, Provider.PROVIDER_UNKNOWN);
switch (providerName) {
case "CDF" -> {
FileChooser fc = new FileChooser();
fc.setInitialDirectory(new File(FileUtils.getUserDirectory()));
fc.getExtensionFilters().add(new ExtensionFilter("JSON files", "*.json"));
fc.setTitle("Select CDF Cast Vote Record File");
openFile = fc.showOpenDialog(GuiContext.getInstance().getMainWindow());
break;
}
case "ES&S": {
case "Dominion", "Hart" -> {
tarheel marked this conversation as resolved.
Show resolved Hide resolved
DirectoryChooser dc = new DirectoryChooser();
dc.setInitialDirectory(new File(FileUtils.getUserDirectory()));
dc.setTitle("Select " + providerName + " Cast Vote Record Folder");
openFile = dc.showDialog(GuiContext.getInstance().getMainWindow());
}
case "ES&S" -> {
FileChooser fc = new FileChooser();
fc.setInitialDirectory(new File(FileUtils.getUserDirectory()));
fc.getExtensionFilters()
.add(new ExtensionFilter("Excel files", "*.xls", "*.xlsx"));
fc.setTitle("Select ES&S Cast Vote Record File");
openFile = fc.showOpenDialog(GuiContext.getInstance().getMainWindow());
break;
}
case "Hart": {
DirectoryChooser dc = new DirectoryChooser();
dc.setInitialDirectory(new File(FileUtils.getUserDirectory()));
dc.setTitle("Select Hart Cast Vote Record Folder");
openFile = dc.showDialog(GuiContext.getInstance().getMainWindow());
break;
}
default:
default -> {
// Do nothing for unhandled providers
}
}

if (openFile != null) {
Expand Down Expand Up @@ -805,7 +804,7 @@ public LocalDate fromString(String string) {
textFieldCvrFirstVoteRow.setDisable(false);
textFieldCvrIdCol.setDisable(false);
textFieldCvrPrecinctCol.setDisable(false);
} else if (provider.equals("CDF") || provider.equals("Hart")) {
} else if (provider.equals("CDF") || provider.equals("Dominion") || provider.equals("Hart")) {
buttonAddCvrFile.setDisable(false);
textFieldCvrFilePath.setDisable(false);
buttonCvrFilePath.setDisable(false);
Expand Down
15 changes: 11 additions & 4 deletions src/main/java/network/brightspots/rcv/ResultsWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,15 +494,22 @@ void generateOverallSummaryFiles(
generateSummaryJson(roundTallies, tallyTransfers, null, outputPath);
}

// write CastVoteRecords for all contests to the provided folder
// write CastVoteRecords for all contests (or the one specified) to the provided folder
// returns a list of files written
List<String> writeGenericCvrCsv(List<CastVoteRecord> castVoteRecords,
List<String> writeGenericCvrCsv(
List<CastVoteRecord> castVoteRecords,
Collection<Contest> contests,
String csvOutputFolder)
throws IOException {
String csvOutputFolder,
String contestId
) throws IOException {
List<String> filesWritten = new ArrayList<>();
try {
for (Contest contest : contests) {
if (!isNullOrBlank(contestId) && !contest.getId().equals(contestId)) {
HEdingfield marked this conversation as resolved.
Show resolved Hide resolved
// We already skipped loading CVRs for the other contests. This just ensures that we
// don't generate empty CSVs for them.
continue;
}
Path outputPath = Paths.get(
getOutputFilePath(csvOutputFolder, "dominion_conversion_contest", timestampString,
contest.getId()) + ".csv");
Expand Down
40 changes: 30 additions & 10 deletions src/main/java/network/brightspots/rcv/TabulatorSession.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class TabulatorSession {
private final Set<String> precinctIds = new HashSet<>();
private final String timestampString;
private String outputPath;
private List<String> convertedFilesWritten;

TabulatorSession(String configPath) {
this.configPath = configPath;
Expand All @@ -72,21 +73,21 @@ class TabulatorSession {
// read Dominion cvr json into CastVoteRecords
// write CastVoteRecords to generic cvr csv files: one per contest
// return list of files written or null if there was a problem
List<String> convertDominionCvrJsonToGenericCsv(String dominionDataFolder) {
void convertDominionCvrJsonToGenericCsv(String dominionDataFolder) {
DominionCvrReader dominionCvrReader = new DominionCvrReader(dominionDataFolder);
List<CastVoteRecord> castVoteRecords = new ArrayList<>();
List<String> filesWritten;
try {
dominionCvrReader.readCastVoteRecords(castVoteRecords);
dominionCvrReader.readCastVoteRecords(castVoteRecords, null);
ResultsWriter writer = new ResultsWriter().setTimestampString(timestampString);
filesWritten = writer
.writeGenericCvrCsv(castVoteRecords, dominionCvrReader.getContests().values(),
dominionDataFolder);
dominionDataFolder, null);
} catch (Exception exception) {
Logger.log(Level.SEVERE, "Failed to convert Dominion CVR to CSV:\n%s", exception.toString());
filesWritten = null;
}
return filesWritten;
this.convertedFilesWritten = filesWritten;
}

// Visible for testing
Expand All @@ -101,6 +102,10 @@ String getTimestampString() {
return timestampString;
}

// Visible for testing
@SuppressWarnings("unused")
List<String> getConvertedFilesWritten() { return convertedFilesWritten; }

// special mode to just export the CVR as CDF JSON instead of tabulating
void convertToCdf() {
ContestConfig config = ContestConfig.loadContestConfig(configPath);
Expand Down Expand Up @@ -269,20 +274,35 @@ private List<CastVoteRecord> parseCastVoteRecords(ContestConfig config, Set<Stri
// At each iteration of the following loop, we add records from another source file.
for (RawContestConfig.CvrSource source : config.rawConfig.cvrFileSources) {
String cvrPath = config.resolveConfigPath(source.getFilePath());
Provider provider = ContestConfig.getProvider(source);
try {
if (ContestConfig.isCdf(source)) {
Logger.log(Level.INFO, "Reading CDF cast vote record file: %s...", cvrPath);
CommonDataFormatReader reader = new CommonDataFormatReader(cvrPath, config);
reader.parseCvrFile(castVoteRecords);
new CommonDataFormatReader(cvrPath, config).parseCvrFile(castVoteRecords);
continue;
} else if (provider == Provider.DOMINION) {
Logger.log(Level.INFO, "Reading Dominion cast vote records from folder: %s...", cvrPath);
DominionCvrReader reader = new DominionCvrReader(cvrPath);
reader.readCastVoteRecords(castVoteRecords, config.getContestId());
// Before we tabulate, we output a converted generic CSV for the CVRs.
try {
ResultsWriter writer = new ResultsWriter().setTimestampString(timestampString);
this.convertedFilesWritten = writer.writeGenericCvrCsv(
castVoteRecords,
reader.getContests().values(),
config.getOutputDirectory(),
config.getContestId());
} catch (IOException e) {
// error already logged in ResultsWriter
}
continue;
} else if (ContestConfig.getProvider(source) == Provider.ESS) {
} else if (provider == Provider.ESS) {
Logger.log(Level.INFO, "Reading ES&S cast vote record file: %s...", cvrPath);
new StreamingCvrReader(config, source).parseCvrFile(castVoteRecords, precinctIds);
continue;
} else if (ContestConfig.getProvider(source) == Provider.HART) {
HartCvrReader reader = new HartCvrReader(cvrPath, config);
} else if (provider == Provider.HART) {
Logger.log(Level.INFO, "Reading Hart cast vote records from folder: %s...", cvrPath);
reader.readCastVoteRecordsFromFolder(castVoteRecords);
new HartCvrReader(cvrPath, config).readCastVoteRecordsFromFolder(castVoteRecords);
continue;
}
throw new UnrecognizedProviderException();
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/network/brightspots/rcv/config_file_documentation.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ Config file must be valid JSON format. Examples can be found in the test_data fo
example: "Portland Mayoral Race 2017"
value: text string of length [1..1000]

"contestId" required when CVR source files are from Hart; must be blank otherwise
"contestId" required when CVR source files are from Dominion or Hart; must be blank otherwise
the ID of the contest to tabulate, as represented in the CVR file(s)
example: "b651b997-417a-46d9-a676-a43d4df94ddc"
value: text string of length [1..1000]
Expand Down
23 changes: 19 additions & 4 deletions src/test/java/network/brightspots/rcv/TabulatorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import java.util.logging.Level;
import network.brightspots.rcv.ContestConfig.Provider;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -114,14 +114,14 @@ private static String getTestFilePath(String stem, String suffix) {
.toString();
}

// helper function test Dominion CVR conversion routine
// helper function to test Dominion CVR conversion routine
private static void runDominionCvrConversionTest(String stem) {
String dominionDataFolder = Paths.get(System.getProperty("user.dir"), TEST_ASSET_FOLDER, stem)
.toAbsolutePath().toString();
TabulatorSession session = new TabulatorSession(null);
List<String> filesWritten = session.convertDominionCvrJsonToGenericCsv(dominionDataFolder);
session.convertDominionCvrJsonToGenericCsv(dominionDataFolder);

for (String convertedFile : filesWritten) {
for (String convertedFile : session.getConvertedFilesWritten()) {
String contestNumber = convertedFile
.substring(convertedFile.lastIndexOf('_') + 1, convertedFile.lastIndexOf('.'));
String expectedPath = Paths
Expand Down Expand Up @@ -155,6 +155,15 @@ private static void runTabulationTest(String stem) {
compareJsons(config, stem, timestampString, null);
}

// If this is a Dominion tabulation test, also check the converted output file.
boolean isDominion = config.rawConfig.cvrFileSources.stream().anyMatch(source ->
HEdingfield marked this conversation as resolved.
Show resolved Hide resolved
ContestConfig.getProvider(source) == Provider.DOMINION);
if (isDominion) {
String expectedPath = getTestFilePath(stem,
"_contest_" + config.getContestId() + "_expected.csv");
assertTrue(fileCompare(session.getConvertedFilesWritten().get(0), expectedPath));
}

// test passed so cleanup test output folder
File outputFolder = new File(session.getOutputPath());
if (outputFolder.listFiles() != null) {
Expand Down Expand Up @@ -235,6 +244,12 @@ void testDominionCvrConversionWyoming() {
runDominionCvrConversionTest("dominion_cvr_conversion_wyoming");
}

@Test
@DisplayName("Dominion direct tabulation test - Alaska test data")
void testDominionDirectTabulationAlaska() {
runTabulationTest("dominion_direct_tabulation_alaska");
}

@Test
@DisplayName("test invalid params in config file")
void invalidParamsTest() {
Expand Down
4 changes: 2 additions & 2 deletions ...htspots/rcv/test_data/2013_minneapolis_mayor/2013_minneapolis_mayor_expected_summary.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@
"STEPHANIE WOODRUFF" : "1012",
"TONY LANE" : "219",
"TROY BENJEGERDES" : "149",
"UWI" : "117"
"Undeclared Write-ins" : "117"
},
"tallyResults" : [ {
"eliminated" : "UWI",
"eliminated" : "Undeclared Write-ins",
"transfers" : {
"ABDUL M RAHAMAN THE ROCK" : "1",
"ALICIA K. BENNETT" : "1",
Expand Down
4 changes: 2 additions & 2 deletions ...test_data/2013_minneapolis_mayor_scale/2013_minneapolis_mayor_scale_expected_summary.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@
"STEPHANIE WOODRUFF" : "13156",
"TONY LANE" : "2847",
"TROY BENJEGERDES" : "1937",
"UWI" : "1521"
"Undeclared Write-ins" : "1521"
},
"tallyResults" : [ {
"eliminated" : "UWI",
"eliminated" : "Undeclared Write-ins",
"transfers" : {
"ABDUL M RAHAMAN THE ROCK" : "13",
"ALICIA K. BENNETT" : "13",
Expand Down
6 changes: 3 additions & 3 deletions ...ightspots/rcv/test_data/2013_minneapolis_park/2013_minneapolis_park_expected_summary.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
"MEG FORNEY" : "7856",
"STEVE BARLAND" : "3705",
"TOM NORDYKE" : "6511",
"UWI" : "342"
"Undeclared Write-ins" : "342"
},
"tallyResults" : [ {
"eliminated" : "UWI",
"eliminated" : "Undeclared Write-ins",
"transfers" : {
"ANNIE YOUNG" : "8",
"CASPER HILL" : "4",
Expand Down Expand Up @@ -235,4 +235,4 @@
},
"tallyResults" : [ ]
} ]
}
}
4 changes: 2 additions & 2 deletions ...a/2013_minneapolis_park_bottoms_up/2013_minneapolis_park_bottoms_up_expected_summary.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
"MEG FORNEY" : "7856",
"STEVE BARLAND" : "3705",
"TOM NORDYKE" : "6511",
"UWI" : "342"
"Undeclared Write-ins" : "342"
},
"tallyResults" : [ {
"eliminated" : "UWI",
"eliminated" : "Undeclared Write-ins",
"transfers" : {
"ANNIE YOUNG" : "8",
"CASPER HILL" : "4",
Expand Down
Loading