Skip to content

Commit

Permalink
Merge pull request #248 from veraPDF/merge/multi-threaded
Browse files Browse the repository at this point in the history
FEAT - Multithreaded processing
  • Loading branch information
carlwilson committed Apr 25, 2018
2 parents f53e1bb + 630fb56 commit fbad8d6
Show file tree
Hide file tree
Showing 10 changed files with 524 additions and 46 deletions.
@@ -0,0 +1,74 @@
package org.verapdf.apps;

import org.junit.Before;
import org.junit.Test;
import org.verapdf.pdfa.VeraGreenfieldFoundryProvider;
import org.verapdf.pdfa.results.ValidationResult;
import org.verapdf.pdfa.validation.validators.CallableValidatorForTest;

import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import static junit.framework.TestCase.assertTrue;

public class MultithreadingTest {
private static final String TEST_FILE = "src/test/resources/veraPDFtest-pass-a.pdf";

@Before
public void init() {
VeraGreenfieldFoundryProvider.initialise();
}

@Test
public void shouldCreateXmlReportsInGivenDirectory() throws Exception {
File fileToValidate = new File(TEST_FILE);

int numberOfThreads = 4;

List<Future<ValidationResult>> futureResult = startValidation(fileToValidate, numberOfThreads);
List<ValidationResult> validationResults = getValidationResult(futureResult);

Boolean isExpectedResults = compareResultsFromDifferentThreads(validationResults);

assertTrue(isExpectedResults);
}

private boolean compareResultsFromDifferentThreads(List<ValidationResult> validationResults) {
boolean isExpectedResults = true;
for (ValidationResult result : validationResults) {
for (ValidationResult comparingResult : validationResults) {
boolean isCompliantResultsEquals = result.isCompliant() == comparingResult.isCompliant();
boolean isTestAssertionsEquals = result.getTestAssertions().equals(comparingResult.getTestAssertions());
boolean isTotalAssertionsEquals = result.getTotalAssertions() == comparingResult.getTotalAssertions();
if (!isCompliantResultsEquals || !isTestAssertionsEquals || !isTotalAssertionsEquals) {
isExpectedResults = false;
}
}
}
return isExpectedResults;
}

private List<ValidationResult> getValidationResult(List<Future<ValidationResult>> results) throws InterruptedException, java.util.concurrent.ExecutionException {
List<ValidationResult> validationResults = new ArrayList<>();
for (Future<ValidationResult> result : results) {
ValidationResult validationResult = result.get();
validationResults.add(validationResult);
}
return validationResults;
}

private List<Future<ValidationResult>> startValidation(File fileToValidate, int numberOfThreads) {
List<Future<ValidationResult>> futureResult = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
for (int i = 0; i < numberOfThreads; i++) {
CallableValidatorForTest validator = new CallableValidatorForTest(fileToValidate);
Future<ValidationResult> submit = executor.submit(validator);
futureResult.add(submit);
}
return futureResult;
}
}
@@ -0,0 +1,29 @@
package org.verapdf.pdfa.validation.validators;

import org.verapdf.pdfa.Foundries;
import org.verapdf.pdfa.PDFAParser;
import org.verapdf.pdfa.PDFAValidator;
import org.verapdf.pdfa.results.ValidationResult;

import java.io.File;
import java.io.FileInputStream;
import java.util.concurrent.Callable;

public class CallableValidatorForTest implements Callable<ValidationResult> {
private File fileToValidate;

public CallableValidatorForTest(File fileToValidate) {
this.fileToValidate = fileToValidate;
}

@Override
public ValidationResult call() throws Exception {
ValidationResult result;
try (FileInputStream fis = new FileInputStream(this.fileToValidate);
PDFAParser parser = Foundries.defaultInstance().createParser(fis);
PDFAValidator validator = Foundries.defaultInstance().createValidator(parser.getFlavour(), false)) {
result = validator.validate(parser);
}
return result;
}
}
Binary file not shown.
30 changes: 28 additions & 2 deletions gui/src/main/java/org/verapdf/apps/Applications.java
@@ -1,6 +1,6 @@
/**
* This file is part of VeraPDF Library GUI, a module of the veraPDF project.
* Copyright (c) 2015, veraPDF Consortium <info@verapdf.org> All rights
* Copyright (c) 2015, veraPDF Consortium <info@VERAPDF.org> All rights
* reserved. VeraPDF Library GUI is free software: you can redistribute it
* and/or modify it under the terms of either: The GNU General public license
* GPLv3+. You should have received a copy of the GNU General Public License
Expand Down Expand Up @@ -46,11 +46,37 @@ public final class Applications {
public static final String UPDATE_SERVICE_NOT_AVAILABLE = "Update Service not available"; //$NON-NLS-1$
public static final String UPDATE_LATEST_VERSION = "You are currently running the latest version of veraPDF%s v%s"; //$NON-NLS-1$
public static final String UPDATE_OLD_VERSION = "You are NOT running the latest version of veraPDF.\nYou are running version %s, the latest version is %s.\n"; //$NON-NLS-1$
public static final String UPDATE_URI = "http://downloads.verapdf.org/rel/verapdf-installer.zip"; //$NON-NLS-1$
public static final String UPDATE_URI = "http://downloads.VERAPDF.org/rel/VERAPDF-installer.zip"; //$NON-NLS-1$

private static final String VERAPDF = "verapdf";
private static final String BAT_EXTENSION = ".bat";

private Applications() {
assert (false);
}

public static File getVeraScriptFile() {
File startFile = null;
String appHome = System.getProperty(APP_HOME_PROPERTY);

if (appHome != null) {
File veraPdfDirectory = new File(appHome);
startFile = getStartFile(veraPdfDirectory);
}
return startFile;
}

private static File getStartFile(File veraPdfDirectory) {
File unixStartFile = new File(veraPdfDirectory, VERAPDF);
if (unixStartFile.isFile()) {
return unixStartFile;
}
File windowsStartFile = new File(veraPdfDirectory, VERAPDF+BAT_EXTENSION);
if (windowsStartFile.isFile()) {
return windowsStartFile;
}
return null;
}

/**
* @return the Application Release details for the GUI
Expand Down
70 changes: 54 additions & 16 deletions gui/src/main/java/org/verapdf/cli/VeraPdfCli.java
Expand Up @@ -17,10 +17,14 @@
*/
package org.verapdf.cli;

import java.io.*;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Scanner;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -29,6 +33,7 @@
import org.verapdf.apps.ConfigManager;
import org.verapdf.apps.SoftwareUpdater;
import org.verapdf.cli.commands.VeraCliArgParser;
import org.verapdf.cli.multithread.MultiThreadProcessor;
import org.verapdf.core.VeraPDFException;
import org.verapdf.pdfa.flavours.PDFAFlavour;
import org.verapdf.pdfa.validation.profiles.ProfileDirectory;
Expand All @@ -49,16 +54,17 @@ public final class VeraPdfCli {
private static final String FLAVOURS_HEADING = CliConstants.APP_NAME + " supported PDF/A profiles:"; //$NON-NLS-1$
private static final ProfileDirectory PROFILES = Profiles.getVeraProfileDirectory();

public static final String EXIT = "q";

private VeraPdfCli() {
// disable default constructor
}

/**
* Main CLI entry point, process the command line arguments
*
* @param args
* Java.lang.String array of command line args, to be processed
* using Apache commons CLI.
* @param args Java.lang.String array of command line args, to be processed
* using Apache commons CLI.
*/
public static void main(final String[] args) throws VeraPDFException {
MemoryMXBean memoryMan = ManagementFactory.getMemoryMXBean();
Expand All @@ -73,24 +79,22 @@ public static void main(final String[] args) throws VeraPDFException {
jCommander.parse(args);
} catch (ParameterException e) {
System.err.println(e.getMessage());
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(1);
displayHelpAndExit(cliArgParser, jCommander, 1);
}
if (cliArgParser.isHelp()) {
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(0);
displayHelpAndExit(cliArgParser, jCommander, 0);
}
messagesFromParser(cliArgParser);
if (isProcess(cliArgParser)) {
try (VeraPdfCliProcessor processor = VeraPdfCliProcessor.createProcessorFromArgs(cliArgParser,
configManager)) {
if (args.length == 0)
jCommander.usage();
// FIXME: trap policy IO Exception (deliberately left un-caught
// for development)
processor.processPaths(cliArgParser.getPdfPaths());
if (args.length == 0) {
jCommander.usage();
}
try {
if (cliArgParser.isServerMode() || cliArgParser.getNumberOfProcesses() < 2) {
singleThreadProcess(cliArgParser);
} else {
MultiThreadProcessor.process(cliArgParser);
}
} catch (OutOfMemoryError oome) {
final String message = "The JVM appears to have run out of memory"; //$NON-NLS-1$
logger.log(Level.WARNING, message, oome);
Expand All @@ -111,6 +115,40 @@ public static void main(final String[] args) throws VeraPDFException {
}
}

private static void singleThreadProcess(VeraCliArgParser cliArgParser) throws VeraPDFException {
try (VeraPdfCliProcessor processor = VeraPdfCliProcessor.createProcessorFromArgs(cliArgParser,
configManager)) {
// FIXME: trap policy IO Exception (deliberately left un-caught for development)
processor.processPaths(cliArgParser.getPdfPaths());
if (cliArgParser.isServerMode()) {
File tempFile = processor.getTempFile();
if (tempFile != null) {
System.out.println(tempFile.getAbsoluteFile());
}
Scanner scanner = new Scanner(System.in);
while (scanner.hasNextLine()) {
String path = scanner.nextLine();
if (path != null) {
if (path.equals(EXIT)) {
break;
} else {
List<String> pathes = new ArrayList<>();
pathes.add(path);
processor.processPaths(pathes);
System.out.println(processor.getTempFile().getAbsolutePath());
}
}
}
}
}
}

public static void displayHelpAndExit(VeraCliArgParser cliArgParser, JCommander jCommander, int i) {
showVersionInfo(cliArgParser.isVerbose());
jCommander.usage();
System.exit(i);
}

private static void messagesFromParser(final VeraCliArgParser parser) {

if (parser.listProfiles()) {
Expand Down
31 changes: 26 additions & 5 deletions gui/src/main/java/org/verapdf/cli/VeraPdfCliProcessor.java
Expand Up @@ -18,6 +18,7 @@
package org.verapdf.cli;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand Down Expand Up @@ -51,16 +52,21 @@ final class VeraPdfCliProcessor implements Closeable {
private final VeraAppConfig appConfig;
private final boolean isPolicy;
private final boolean isRecursive;
private final boolean isServerMode;
private final File tempMrrFile;
private final File policyFile;
private boolean isStdOut = true;
private boolean appendData = true;
private String baseDirectory = ""; //$NON-NLS-1$
private OutputStream os;
private File tempFile;

private VeraPdfCliProcessor(final VeraCliArgParser args, ConfigManager configManager) throws VeraPDFException {
this.configManager = configManager;
this.isPolicy = args.isPolicy();
this.isRecursive = args.isRecurse();
this.isServerMode = args.isServerMode();

try {
this.tempMrrFile = (this.isPolicy) ? File.createTempFile("mrr", "veraPDF") : null; //$NON-NLS-1$//$NON-NLS-2$
} catch (IOException excep) {
Expand Down Expand Up @@ -93,6 +99,16 @@ ProcessorConfig getProcessorConfig() {
}

void processPaths(final List<String> pdfPaths) throws VeraPDFException {
if (isServerMode) {
try {
this.tempFile = Files.createTempFile("tempReport", ".xml").toFile();
this.os = new FileOutputStream(tempFile);
} catch (IOException e) {
logger.log(Level.SEVERE, "Can't create temp file", e);
}
} else {
this.os = System.out;
}
// If the path list is empty then process the STDIN stream
if (pdfPaths.isEmpty()) {
processStdIn();
Expand Down Expand Up @@ -135,6 +151,7 @@ private void processFilePaths(final List<String> paths) {
ProcessorFactory.getHandler(this.appConfig.getFormat(), this.appConfig.isVerbose(), reportStream,
this.appConfig.getMaxFailsDisplayed(),
this.processorConfig.getValidatorConfig().isRecordPasses()));
reportStream.flush();
} catch (VeraPDFException excep) {
String message = CliConstants.EXCEP_VERA_BATCH;
System.err.println(message);
Expand All @@ -151,9 +168,9 @@ private void processStream(final ItemDetails item, final InputStream toProcess)

OutputStream outputReportStream = this.getReportStream();
try {
if (result.isPdf() && !result.isEncryptedPdf())
if (result.isPdf() && !result.isEncryptedPdf()) {
ProcessorFactory.resultToXml(result, outputReportStream, true);
else {
} else {
String message = String.format(
(result.isPdf()) ? CliConstants.MESS_PDF_ENCRYPTED : CliConstants.MESS_PDF_NOT_VALID,
item.getName());
Expand Down Expand Up @@ -188,11 +205,11 @@ private OutputStream getReportStream() {
throw new IllegalStateException("Policy enabled BUT no temp destination", excep);
}
}
return System.out;
return this.os;
}

private void applyPolicy() throws VeraPDFException {
File tempPolicyResult = null;
File tempPolicyResult;
try {
tempPolicyResult = File.createTempFile("policyResult", "veraPDF");
} catch (IOException excep) {
Expand All @@ -201,7 +218,7 @@ private void applyPolicy() throws VeraPDFException {
try (InputStream mrrIs = new FileInputStream(this.tempMrrFile);
OutputStream policyResultOs = new FileOutputStream(tempPolicyResult)) {
PolicyChecker.applyPolicy(this.policyFile, mrrIs, policyResultOs);
PolicyChecker.insertPolicyReport(tempPolicyResult, this.tempMrrFile, System.out);
PolicyChecker.insertPolicyReport(tempPolicyResult, this.tempMrrFile, os);
} catch (FileNotFoundException excep) {
throw new VeraPDFException("Could not find temporary policy result file.", excep);
} catch (IOException excep) {
Expand Down Expand Up @@ -260,4 +277,8 @@ public void close() {
this.tempMrrFile.deleteOnExit();
}
}

public File getTempFile() {
return tempFile;
}
}

0 comments on commit fbad8d6

Please sign in to comment.