From 91eaf88c10e7b2ade09291d0b09627de0c5f3045 Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Oct 2017 14:34:19 -0400 Subject: [PATCH 1/2] Catching and handling exceptions caused by bad ZK node data so that they don't stop input folder scans --- .../autoingest/AutoIngestJob.java | 132 +++++++++++------- .../autoingest/AutoIngestManager.java | 22 +-- .../autoingest/AutoIngestMonitor.java | 2 +- 3 files changed, 99 insertions(+), 57 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java index 5e9bfbd26cb..dd4cb60377b 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestJob.java @@ -89,30 +89,34 @@ public final class AutoIngestJob implements Comparable, Serializa * * @param manifest The manifest for an automated ingest job. */ - AutoIngestJob(Manifest manifest) { - /* - * Version 0 fields. - */ - this.manifest = manifest; - this.nodeName = ""; - this.caseDirectoryPath = ""; - this.priority = DEFAULT_PRIORITY; - this.stage = Stage.PENDING; - this.stageStartDate = manifest.getDateFileCreated(); - this.dataSourceProcessor = null; - this.ingestJob = null; - this.cancelled = false; - this.completed = false; - this.completedDate = new Date(0); - this.errorsOccurred = false; - - /* - * Version 1 fields. - */ - this.version = CURRENT_VERSION; - this.processingStatus = ProcessingStatus.PENDING; - this.numberOfCrashes = 0; - this.stageDetails = this.getProcessingStageDetails(); + AutoIngestJob(Manifest manifest) throws AutoIngestJobException { + try { + /* + * Version 0 fields. + */ + this.manifest = manifest; + this.nodeName = ""; + this.caseDirectoryPath = ""; + this.priority = DEFAULT_PRIORITY; + this.stage = Stage.PENDING; + this.stageStartDate = manifest.getDateFileCreated(); + this.dataSourceProcessor = null; + this.ingestJob = null; + this.cancelled = false; + this.completed = false; + this.completedDate = new Date(0); + this.errorsOccurred = false; + + /* + * Version 1 fields. + */ + this.version = CURRENT_VERSION; + this.processingStatus = ProcessingStatus.PENDING; + this.numberOfCrashes = 0; + this.stageDetails = this.getProcessingStageDetails(); + } catch (Exception ex) { + throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex); + } } /** @@ -122,30 +126,34 @@ public final class AutoIngestJob implements Comparable, Serializa * @param nodeData The coordination service node data for an automated * ingest job. */ - AutoIngestJob(AutoIngestJobNodeData nodeData) { - /* - * Version 0 fields. - */ - this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap()); - this.nodeName = nodeData.getProcessingHostName(); - this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString(); - this.priority = nodeData.getPriority(); - this.stage = nodeData.getProcessingStage(); - this.stageStartDate = nodeData.getProcessingStageStartDate(); - this.dataSourceProcessor = null; // Transient data not in node data. - this.ingestJob = null; // Transient data not in node data. - this.cancelled = false; // Transient data not in node data. - this.completed = false; // Transient data not in node data. - this.completedDate = nodeData.getCompletedDate(); - this.errorsOccurred = nodeData.getErrorsOccurred(); - - /* - * Version 1 fields. - */ - this.version = CURRENT_VERSION; - this.processingStatus = nodeData.getProcessingStatus(); - this.numberOfCrashes = nodeData.getNumberOfCrashes(); - this.stageDetails = this.getProcessingStageDetails(); + AutoIngestJob(AutoIngestJobNodeData nodeData) throws AutoIngestJobException { + try { + /* + * Version 0 fields. + */ + this.manifest = new Manifest(nodeData.getManifestFilePath(), nodeData.getManifestFileDate(), nodeData.getCaseName(), nodeData.getDeviceId(), nodeData.getDataSourcePath(), Collections.emptyMap()); + this.nodeName = nodeData.getProcessingHostName(); + this.caseDirectoryPath = nodeData.getCaseDirectoryPath().toString(); + this.priority = nodeData.getPriority(); + this.stage = nodeData.getProcessingStage(); + this.stageStartDate = nodeData.getProcessingStageStartDate(); + this.dataSourceProcessor = null; // Transient data not in node data. + this.ingestJob = null; // Transient data not in node data. + this.cancelled = false; // Transient data not in node data. + this.completed = false; // Transient data not in node data. + this.completedDate = nodeData.getCompletedDate(); + this.errorsOccurred = nodeData.getErrorsOccurred(); + + /* + * Version 1 fields. + */ + this.version = CURRENT_VERSION; + this.processingStatus = nodeData.getProcessingStatus(); + this.numberOfCrashes = nodeData.getNumberOfCrashes(); + this.stageDetails = this.getProcessingStageDetails(); + } catch (Exception ex) { + throw new AutoIngestJobException(String.format("Error creating automated ingest job"), ex); + } } /** @@ -622,5 +630,33 @@ Date getStartDate() { } } + + /** + * Exception thrown when there is a problem creating auto ingest job. + */ + final static class AutoIngestJobException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * Constructs an exception to throw when there is a problem creating + * auto ingest job. + * + * @param message The exception message. + */ + private AutoIngestJobException(String message) { + super(message); + } + + /** + * Constructs an exception to throw when there is a problem creating + * auto ingest job. + * + * @param message The exception message. + * @param cause The cause of the exception, if it was an exception. + */ + private AutoIngestJobException(String message, Throwable cause) { + super(message, cause); + } + } } diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index 18d33b5c696..ee709fe544f 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -61,6 +61,7 @@ import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case.CaseType; @@ -93,6 +94,7 @@ import org.sleuthkit.autopsy.experimental.configuration.SharedConfiguration.SharedConfigurationException; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor; import org.sleuthkit.autopsy.datasourceprocessors.AutoIngestDataSourceProcessor.AutoIngestDataSourceProcessorException; +import org.sleuthkit.autopsy.experimental.autoingest.AutoIngestJob.AutoIngestJobException; import org.sleuthkit.autopsy.ingest.IngestJob; import org.sleuthkit.autopsy.ingest.IngestJob.CancellationReason; import org.sleuthkit.autopsy.ingest.IngestJobSettings; @@ -759,7 +761,7 @@ CaseDeletionResult deleteCase(String caseName, Path caseDirectoryPath) { AutoIngestJob deletedJob = new AutoIngestJob(nodeData); deletedJob.setProcessingStatus(AutoIngestJob.ProcessingStatus.DELETED); this.updateCoordinationServiceNode(deletedJob); - } catch (AutoIngestJobNodeData.InvalidDataException ex) { + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); return CaseDeletionResult.PARTIALLY_DELETED; } catch (InterruptedException | CoordinationServiceException ex) { @@ -1088,11 +1090,15 @@ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throw SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); break; } - } catch (AutoIngestJobNodeData.InvalidDataException ex) { - SYS_LOGGER.log(Level.WARNING, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); } } else { - addNewPendingJob(manifest); + try { + addNewPendingJob(manifest); + } catch (AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); + } } } catch (CoordinationServiceException ex) { SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); @@ -1122,7 +1128,7 @@ public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throw * blocked, i.e., if auto ingest is * shutting down. */ - private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException { + private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { AutoIngestJob job; if (nodeData.getVersion() == AutoIngestJobNodeData.getCurrentVersion()) { job = new AutoIngestJob(nodeData); @@ -1176,7 +1182,7 @@ private void addPendingJob(Manifest manifest, AutoIngestJobNodeData nodeData) th * blocked, i.e., if auto ingest is * shutting down. */ - private void addNewPendingJob(Manifest manifest) throws InterruptedException { + private void addNewPendingJob(Manifest manifest) throws InterruptedException, AutoIngestJobException { /* * Create the coordination service node data for the job. Note that * getting the lock will create the node for the job (with no data) @@ -1218,7 +1224,7 @@ private void addNewPendingJob(Manifest manifest) throws InterruptedException { * blocked, i.e., if auto ingest is * shutting down. */ - private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException { + private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeData) throws InterruptedException, AutoIngestJobException { /* * Try to get an exclusive lock on the coordination service node for * the job. If the lock cannot be obtained, another host in the auto @@ -1314,7 +1320,7 @@ private void doRecoveryIfCrashed(Manifest manifest, AutoIngestJobNodeData nodeDa * @throws CoordinationServiceException * @throws InterruptedException */ - private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException { + private void addCompletedJob(Manifest manifest, AutoIngestJobNodeData nodeData) throws CoordinationServiceException, InterruptedException, AutoIngestJobException { Path caseDirectoryPath = PathUtils.findCaseDirectory(rootOutputDirectory, manifest.getCaseName()); if (null != caseDirectoryPath) { AutoIngestJob job; diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java index cec605f06bd..9a262bb6d34 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestMonitor.java @@ -265,7 +265,7 @@ private JobsSnapshot queryCoordinationService() { } } catch (InterruptedException ex) { LOGGER.log(Level.SEVERE, String.format("Unexpected interrupt while retrieving coordination service node data for '%s'", node), ex); - } catch (AutoIngestJobNodeData.InvalidDataException ex) { + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJob.AutoIngestJobException ex) { LOGGER.log(Level.SEVERE, String.format("Unable to use node data for '%s'", node), ex); } } From ca14274ec879df61e572e3b48fdf9f27e4338add Mon Sep 17 00:00:00 2001 From: Eugene Livis Date: Fri, 6 Oct 2017 14:51:28 -0400 Subject: [PATCH 2/2] Added a catch-all firewal to visitFile() so that one bad file doesn't stop the entire input folder scan --- .../autoingest/AutoIngestManager.java | 143 +++++++++--------- 1 file changed, 75 insertions(+), 68 deletions(-) diff --git a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java index ee709fe544f..c5f43a8f1f0 100755 --- a/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java +++ b/Experimental/src/org/sleuthkit/autopsy/experimental/autoingest/AutoIngestManager.java @@ -1017,96 +1017,103 @@ public FileVisitResult preVisitDirectory(Path dirPath, BasicFileAttributes dirAt * @return TERMINATE if auto ingest is shutting down, CONTINUE if it has * not. * - * @throws IOException if an I/O error occurs, but this implementation - * does not throw. */ @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) { if (Thread.currentThread().isInterrupted()) { return TERMINATE; } - Manifest manifest = null; - for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) { - if (parser.fileIsManifest(filePath)) { - try { - manifest = parser.parse(filePath); - break; - } catch (ManifestFileParserException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex); + try { + Manifest manifest = null; + for (ManifestFileParser parser : Lookup.getDefault().lookupAll(ManifestFileParser.class)) { + if (parser.fileIsManifest(filePath)) { + try { + manifest = parser.parse(filePath); + break; + } catch (ManifestFileParserException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error attempting to parse %s with parser %s", filePath, parser.getClass().getCanonicalName()), ex); + } + } + if (Thread.currentThread().isInterrupted()) { + return TERMINATE; } } + if (Thread.currentThread().isInterrupted()) { return TERMINATE; } - } - if (Thread.currentThread().isInterrupted()) { - return TERMINATE; - } - - if (null != manifest) { - /* + if (null != manifest) { + /* * Update the mapping of case names to manifest paths that is * used for case deletion. - */ - String caseName = manifest.getCaseName(); - Path manifestPath = manifest.getFilePath(); - if (casesToManifests.containsKey(caseName)) { - Set manifestPaths = casesToManifests.get(caseName); - manifestPaths.add(manifestPath); - } else { - Set manifestPaths = new HashSet<>(); - manifestPaths.add(manifestPath); - casesToManifests.put(caseName, manifestPaths); - } + */ + String caseName = manifest.getCaseName(); + Path manifestPath = manifest.getFilePath(); + if (casesToManifests.containsKey(caseName)) { + Set manifestPaths = casesToManifests.get(caseName); + manifestPaths.add(manifestPath); + } else { + Set manifestPaths = new HashSet<>(); + manifestPaths.add(manifestPath); + casesToManifests.put(caseName, manifestPaths); + } - /* + /* * Add a job to the pending jobs queue, the completed jobs list, * or do crashed job recovery, as required. - */ - try { - byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); - if (null != rawData && rawData.length > 0) { - try { - AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData); - AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus(); - switch (processingStatus) { - case PENDING: - addPendingJob(manifest, nodeData); - break; - case PROCESSING: - doRecoveryIfCrashed(manifest, nodeData); - break; - case COMPLETED: - addCompletedJob(manifest, nodeData); - break; - case DELETED: - /* + */ + try { + byte[] rawData = coordinationService.getNodeData(CoordinationService.CategoryNode.MANIFESTS, manifestPath.toString()); + if (null != rawData && rawData.length > 0) { + try { + AutoIngestJobNodeData nodeData = new AutoIngestJobNodeData(rawData); + AutoIngestJob.ProcessingStatus processingStatus = nodeData.getProcessingStatus(); + switch (processingStatus) { + case PENDING: + addPendingJob(manifest, nodeData); + break; + case PROCESSING: + doRecoveryIfCrashed(manifest, nodeData); + break; + case COMPLETED: + addCompletedJob(manifest, nodeData); + break; + case DELETED: + /* * Ignore jobs marked as "deleted." - */ - break; - default: - SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); - break; + */ + break; + default: + SYS_LOGGER.log(Level.SEVERE, "Unknown ManifestNodeData.ProcessingStatus"); + break; + } + } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); + } + } else { + try { + addNewPendingJob(manifest); + } catch (AutoIngestJobException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); } - } catch (AutoIngestJobNodeData.InvalidDataException | AutoIngestJobException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Invalid auto ingest job node data for %s", manifestPath), ex); - } - } else { - try { - addNewPendingJob(manifest); - } catch (AutoIngestJobException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Invalid manifest data for %s", manifestPath), ex); } + } catch (CoordinationServiceException ex) { + SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); + return CONTINUE; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return TERMINATE; } - } catch (CoordinationServiceException ex) { - SYS_LOGGER.log(Level.SEVERE, String.format("Error transmitting node data for %s", manifestPath), ex); - return CONTINUE; - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - return TERMINATE; } + + } catch (Exception ex) { + // Catch all unhandled and unexpected exceptions. Otherwise one bad file + // can stop the entire input folder scanning. Given that the exception is unexpected, + // I'm hesitant to add logging which requires accessing or de-referencing data. + SYS_LOGGER.log(Level.SEVERE, "Unexpected exception in file visitor", ex); + return CONTINUE; } if (!Thread.currentThread().isInterrupted()) {