diff --git a/pom.xml b/pom.xml index 1a2755d..27543e5 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ ai.preferred venom - 4.1.3 + 4.1.4-SNAPSHOT jar ${project.groupId}:${project.artifactId} diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java index 8782a83..05f1376 100644 --- a/src/main/java/ai/preferred/venom/Crawler.java +++ b/src/main/java/ai/preferred/venom/Crawler.java @@ -16,10 +16,7 @@ package ai.preferred.venom; -import ai.preferred.venom.fetcher.AsyncFetcher; -import ai.preferred.venom.fetcher.Callback; -import ai.preferred.venom.fetcher.Fetcher; -import ai.preferred.venom.fetcher.StopCodeException; +import ai.preferred.venom.fetcher.*; import ai.preferred.venom.job.AbstractQueueScheduler; import ai.preferred.venom.job.Job; import ai.preferred.venom.job.PriorityQueueScheduler; @@ -28,6 +25,7 @@ import ai.preferred.venom.request.Request; import ai.preferred.venom.response.Response; import ai.preferred.venom.response.VResponse; +import ai.preferred.venom.validator.Validator; import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -706,7 +704,8 @@ public void completed(final Request request, final Response response) { public void failed(final Request request, final Exception ex) { crawler.connections.release(); crawler.threadPool.execute(() -> { - if (ex instanceof StopCodeException) { + if (ex instanceof StopCodeException + || (ex instanceof ValidationException && ((ValidationException) ex).getStatus() == Validator.Status.STOP)) { crawler.pendingJobs.remove(job); } else { synchronized (crawler.pendingJobs) { diff --git a/src/main/java/ai/preferred/venom/fetcher/AsyncResponseConsumer.java b/src/main/java/ai/preferred/venom/fetcher/AsyncResponseConsumer.java index 9963e2e..e053b15 100644 --- a/src/main/java/ai/preferred/venom/fetcher/AsyncResponseConsumer.java +++ b/src/main/java/ai/preferred/venom/fetcher/AsyncResponseConsumer.java @@ -265,7 +265,7 @@ protected final BaseResponse buildResult(final HttpContext context) throws Excep } if (status == Validator.Status.STOP) { - throw new StopCodeException(statusCode, "Validator stopped the request."); + throw new ValidationException(status, response, "Validator stopped the request."); } else if (status != Validator.Status.VALID) { throw new ValidationException(status, response, "Invalid response."); }