From 91751d9e42c74c02c8b071499531811f4a93790b Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Mon, 4 Mar 2019 18:17:38 +0800
Subject: [PATCH 01/17] Separated Scheduler interface

---
 src/main/java/ai/preferred/venom/Crawler.java |  19 ++-
 .../venom/FatalHandlerException.java          |   2 +-
 src/main/java/ai/preferred/venom/Handler.java |   1 -
 .../java/ai/preferred/venom/Scheduler.java    | 111 +++++++++++++++
 .../preferred/venom/fetcher/AsyncFetcher.java |  10 +-
 .../venom/job/AbstractQueueScheduler.java     | 126 +++++++++++++-----
 .../ai/preferred/venom/job/FIFOScheduler.java |  28 +---
 .../ai/preferred/venom/job/LazyScheduler.java |  30 +----
 .../venom/job/PriorityQueueScheduler.java     |  29 +---
 .../ai/preferred/venom/job/Scheduler.java     |  86 +-----------
 .../venom/job/FIFOSchedulerTest.java          |  12 +-
 .../venom/job/LazySchedulerTest.java          |   2 +-
 .../venom/job/PriorityQueueSchedulerTest.java |  14 +-
 13 files changed, 252 insertions(+), 218 deletions(-)
 create mode 100644 src/main/java/ai/preferred/venom/Scheduler.java

diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java
index 61b6031..1399b55 100644
--- a/src/main/java/ai/preferred/venom/Crawler.java
+++ b/src/main/java/ai/preferred/venom/Crawler.java
@@ -17,7 +17,6 @@
 package ai.preferred.venom;
 
 import ai.preferred.venom.fetcher.*;
-import ai.preferred.venom.job.AbstractQueueScheduler;
 import ai.preferred.venom.job.Job;
 import ai.preferred.venom.job.PriorityQueueScheduler;
 import ai.preferred.venom.job.Scheduler;
@@ -89,7 +88,7 @@ public final class Crawler implements Interruptible {
    * The scheduler used.
    */
   @NotNull
-  private final AbstractQueueScheduler scheduler;
+  private final Scheduler<? extends Job> scheduler;
 
   /**
    * The maximum number of simultaneous connections.
@@ -293,8 +292,8 @@ private void run() {
    *
    * @return the instance of scheduler used.
    */
-  public Scheduler getScheduler() {
-    return scheduler;
+  public ai.preferred.venom.Scheduler getScheduler() {
+    return scheduler.getScheduler();
   }
 
   /**
@@ -467,7 +466,7 @@ public static final class Builder {
     /**
      * The scheduler used.
      */
-    private AbstractQueueScheduler scheduler;
+    private Scheduler<? extends Job> scheduler;
 
     /**
      * The sleep scheduler used.
@@ -550,7 +549,7 @@ public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
      * @param scheduler scheduler to be used.
      * @return this
      */
-    public Builder setScheduler(final @NotNull AbstractQueueScheduler scheduler) {
+    public Builder setScheduler(final @NotNull Scheduler<? extends Job> scheduler) {
       this.scheduler = scheduler;
       return this;
     }
@@ -674,13 +673,13 @@ public void completed(final Request request, final Response response) {
       crawler.threadPool.execute(() -> {
         try {
           if (job.getHandler() != null) {
-            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.scheduler, crawler.session,
-                crawler.workerManager.getWorker());
+            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+                crawler.session, crawler.workerManager.getWorker());
           } else if (crawler.router != null) {
             final Handler routedHandler = crawler.router.getHandler(job.getRequest());
             if (routedHandler != null) {
-              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.scheduler, crawler.session,
-                  crawler.workerManager.getWorker());
+              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+                  crawler.session, crawler.workerManager.getWorker());
             }
           } else {
             LOGGER.error("No handler to handle request {}.", job.getRequest().getUrl());
diff --git a/src/main/java/ai/preferred/venom/FatalHandlerException.java b/src/main/java/ai/preferred/venom/FatalHandlerException.java
index e9b1e31..b89ef9d 100644
--- a/src/main/java/ai/preferred/venom/FatalHandlerException.java
+++ b/src/main/java/ai/preferred/venom/FatalHandlerException.java
@@ -3,7 +3,7 @@
 /**
  * This class defines fatal runtime exception for {@link Handler}.
  * If {@link Handler#handle(ai.preferred.venom.request.Request,
- * ai.preferred.venom.response.VResponse, ai.preferred.venom.job.Scheduler,
+ * ai.preferred.venom.response.VResponse, ai.preferred.venom.Scheduler,
  * Session, Worker)} encounters unexpected situation, it can signal
  * {@link Crawler} to stop execution by throwing this exception.
  *
diff --git a/src/main/java/ai/preferred/venom/Handler.java b/src/main/java/ai/preferred/venom/Handler.java
index e7e6702..200be4b 100644
--- a/src/main/java/ai/preferred/venom/Handler.java
+++ b/src/main/java/ai/preferred/venom/Handler.java
@@ -16,7 +16,6 @@
 
 package ai.preferred.venom;
 
-import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.VResponse;
 
diff --git a/src/main/java/ai/preferred/venom/Scheduler.java b/src/main/java/ai/preferred/venom/Scheduler.java
new file mode 100644
index 0000000..46f437b
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/Scheduler.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom;
+
+import ai.preferred.venom.job.Priority;
+import ai.preferred.venom.request.Request;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * This interface represents only the adding part a scheduler.
+ *
+ * @author Maksim Tkachenko
+ * @author Ween Jiann Lee
+ */
+public interface Scheduler {
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to a minimum priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param h  handler to be used to parse the request
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  void add(@NotNull Request r, @NotNull Handler h, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param h handler to be used to parse the request
+   * @param p initial priority of the request
+   */
+  void add(@NotNull Request r, @NotNull Handler h, Priority p);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * it's initialised with default priority that can be downgraded to
+   * the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param h handler to be used to parse the request
+   */
+  void add(@NotNull Request r, @NotNull Handler h);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise, and its priority can be downgraded to a minimum
+   * priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  void add(@NotNull Request r, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise defined, and its priority can be downgraded to the
+   * default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param p initial priority of the request
+   */
+  void add(@NotNull Request r, Priority p);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise defined, and it's initialised with default priority
+   * that can be downgraded to the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   */
+  void add(@NotNull Request r);
+
+}
diff --git a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
index c26c8ef..deac462 100644
--- a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
+++ b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
@@ -133,7 +133,7 @@ public final class AsyncFetcher implements Fetcher {
   private final boolean compressed;
 
   /**
-   * Constructs an instance of async fetcher.
+   * Constructs an instance of AsyncFetcher.
    *
    * @param builder An instance of builder
    */
@@ -186,7 +186,7 @@ private AsyncFetcher(final Builder builder) {
   }
 
   /**
-   * Create an instance of async fetcher with default options.
+   * Create an instance of AsyncFetcher with default options.
    *
    * @return A new instance of async fetcher
    */
@@ -205,10 +205,10 @@ public static Builder builder() {
 
   /**
    * Check if request is an instance of http fetcher request and return it
-   * if true, otherwise wrap it with http fetcher request and return that.
+   * if true, otherwise wrap it with HttpFetcherRequest and return that.
    *
    * @param request An instance of request
-   * @return An instance of http fetcher request
+   * @return An instance of HttpFetcherRequest
    */
   private HttpFetcherRequest normalizeRequest(final Request request) {
     if (request instanceof HttpFetcherRequest) {
@@ -221,7 +221,7 @@ private HttpFetcherRequest normalizeRequest(final Request request) {
    * Prepare fetcher request by prepending headers and set appropriate proxy.
    *
    * @param request An instance of request
-   * @return An instance of http fetcher request
+   * @return An instance of HttpFetcherRequest
    */
   private HttpFetcherRequest prepareFetcherRequest(final Request request) {
     HttpFetcherRequest httpFetcherRequest = normalizeRequest(request);
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index a583427..a9c2540 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -18,6 +18,8 @@
 
 import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.Request;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
 import java.util.AbstractQueue;
@@ -29,79 +31,139 @@
  * @author Ween Jiann Lee
  * @author Maksim Tkachenko
  */
-public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements Scheduler, BlockingQueue<Job> {
+public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements Scheduler<Job> {
 
   /**
-   * Get the queue this instance is using.
-   *
-   * @return An instance of blocking queue
+   * The queue used for this scheduler.
    */
-  abstract BlockingQueue<Job> getQueue();
-
-  @Override
-  public final void add(final Request r, final Handler h, final Priority p) {
-    add(r, h, p, Priority.FLOOR);
-  }
-
-  @Override
-  public final void add(final Request r, final Handler h) {
-    add(r, h, Priority.DEFAULT);
-  }
+  private final BlockingQueue<Job> queue;
 
-  @Override
-  public final void add(final Request r, final Priority p, final Priority pf) {
-    add(r, null, p, pf);
-  }
+  /**
+   * The adding part of the scheduler.
+   */
+  private final ai.preferred.venom.Scheduler scheduler;
 
-  @Override
-  public final void add(final Request r, final Priority p) {
-    add(r, null, p, Priority.FLOOR);
+  /**
+   * Constructs an instance of AbstractQueueScheduler.
+   *
+   * @param queue an instance of BlockingQueue
+   */
+  protected AbstractQueueScheduler(final BlockingQueue<Job> queue) {
+    this.queue = queue;
+    this.scheduler = new ExternalScheduler(queue);
   }
 
   @Override
-  public final void add(final Request r) {
-    add(r, null, Priority.DEFAULT, Priority.FLOOR);
+  public final ai.preferred.venom.Scheduler getScheduler() {
+    return scheduler;
   }
 
   @Nonnull
   @Override
   public final Iterator<Job> iterator() {
-    return getQueue().iterator();
+    return queue.iterator();
   }
 
   @Override
   public final int size() {
-    return getQueue().size();
+    return queue.size();
   }
 
   @Nonnull
   @Override
   public final Job take() throws InterruptedException {
-    return getQueue().take();
+    return queue.take();
   }
 
   @Override
   public final int remainingCapacity() {
-    return getQueue().remainingCapacity();
+    return queue.remainingCapacity();
   }
 
   @Override
   public final int drainTo(final @Nonnull Collection<? super Job> c) {
-    return getQueue().drainTo(c);
+    return queue.drainTo(c);
   }
 
   @Override
   public final int drainTo(final @Nonnull Collection<? super Job> c, final int maxElements) {
-    return getQueue().drainTo(c, maxElements);
+    return queue.drainTo(c, maxElements);
   }
 
   @Override
   public final boolean offer(final @Nonnull Job job) {
-    return getQueue().offer(job);
+    return queue.offer(job);
   }
 
   @Override
   public final Job peek() {
-    return getQueue().peek();
+    return queue.peek();
+  }
+
+  /**
+   * Get the BlockingQueue backing this scheduler.
+   *
+   * @return an instance of BlockingQueue
+   */
+  protected final BlockingQueue<Job> getQueue() {
+    return queue;
+  }
+
+  /**
+   * An implementation of ai.preferred.venom.Scheduler using BasicJob.
+   */
+  public static class ExternalScheduler implements ai.preferred.venom.Scheduler {
+
+    /**
+     * Logger.
+     */
+    private static final Logger LOGGER = LoggerFactory.getLogger(ExternalScheduler.class);
+
+    /**
+     * The queue used for this scheduler.
+     */
+    private final BlockingQueue<Job> queue;
+
+    /**
+     * Constructs an instance of ExternalScheduler
+     *
+     * @param queue an instance of BlockingQueue
+     */
+    public ExternalScheduler(final BlockingQueue<Job> queue) {
+      this.queue = queue;
+    }
+
+    @Override
+    public final void add(final Request r, final Handler h, final Priority p, final Priority pf) {
+      final Job job = new BasicJob(r, h, p, pf, queue);
+      queue.add(job);
+      LOGGER.debug("Job {} - {} added to queue.", job.toString(), r.getUrl());
+    }
+
+    @Override
+    public final void add(final Request r, final Handler h, final Priority p) {
+      add(r, h, p, Priority.FLOOR);
+    }
+
+    @Override
+    public final void add(final Request r, final Handler h) {
+      add(r, h, Priority.DEFAULT);
+    }
+
+    @Override
+    public final void add(final Request r, final Priority p, final Priority pf) {
+      add(r, null, p, pf);
+    }
+
+    @Override
+    public final void add(final Request r, final Priority p) {
+      add(r, null, p, Priority.FLOOR);
+    }
+
+    @Override
+    public final void add(final Request r) {
+      add(r, null, Priority.DEFAULT, Priority.FLOOR);
+    }
+
   }
 }
diff --git a/src/main/java/ai/preferred/venom/job/FIFOScheduler.java b/src/main/java/ai/preferred/venom/job/FIFOScheduler.java
index d0fd080..36f2c5f 100644
--- a/src/main/java/ai/preferred/venom/job/FIFOScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/FIFOScheduler.java
@@ -16,14 +16,7 @@
 
 package ai.preferred.venom.job;
 
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import javax.annotation.Nonnull;
-import javax.validation.constraints.NotNull;
-import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
@@ -39,25 +32,10 @@
 public class FIFOScheduler extends AbstractQueueScheduler {
 
   /**
-   * Logger.
+   * Constructs an instance of FIFOScheduler.
    */
-  private static final Logger LOGGER = LoggerFactory.getLogger(FIFOScheduler.class);
-
-  /**
-   * The queue used for this scheduler.
-   */
-  private final LinkedBlockingQueue<Job> queue = new LinkedBlockingQueue<>();
-
-  @Override
-  final BlockingQueue<Job> getQueue() {
-    return queue;
-  }
-
-  @Override
-  public final void add(final @NotNull Request r, final @NotNull Handler h, final Priority p, final Priority pf) {
-    Job job = new BasicJob(r, h, p, pf, queue);
-    getQueue().add(job);
-    LOGGER.debug("Job {} - {} added to queue.", job.toString(), r.getUrl());
+  public FIFOScheduler() {
+    super(new LinkedBlockingQueue<>());
   }
 
   @Override
diff --git a/src/main/java/ai/preferred/venom/job/LazyScheduler.java b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
index f6aaba3..ae25c36 100644
--- a/src/main/java/ai/preferred/venom/job/LazyScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
@@ -19,8 +19,6 @@
 
 import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
 import java.util.Iterator;
@@ -40,16 +38,6 @@
  */
 public class LazyScheduler extends AbstractQueueScheduler {
 
-  /**
-   * Logger.
-   */
-  private static final Logger LOGGER = LoggerFactory.getLogger(LazyScheduler.class);
-
-  /**
-   * The queue used for this scheduler.
-   */
-  private final PriorityBlockingQueue<Job> queue = new PriorityBlockingQueue<>();
-
   /**
    * An object to synchronise upon.
    */
@@ -72,6 +60,7 @@ public class LazyScheduler extends AbstractQueueScheduler {
    * @param handler  The default handler to use
    */
   public LazyScheduler(final Iterator<Request> requests, final Handler handler) {
+    super(new PriorityBlockingQueue<>());
     this.requests = requests;
     this.handler = handler;
   }
@@ -85,18 +74,6 @@ public LazyScheduler(final Iterator<Request> requests) {
     this(requests, null);
   }
 
-  @Override
-  final PriorityBlockingQueue<Job> getQueue() {
-    return queue;
-  }
-
-  @Override
-  public final void add(final Request r, final Handler h, final Priority p, final Priority pf) {
-    Job job = new BasicJob(r, h, p, pf, queue);
-    getQueue().add(job);
-    LOGGER.debug("Job {} - {} added to queue.", job.toString(), r.getUrl());
-  }
-
   /**
    * Poll request from the iterator.
    *
@@ -117,12 +94,13 @@ public final Job poll() {
   }
 
   @Override
-  public final void put(final @Nonnull Job job) {
+  public final void put(final @Nonnull Job job) throws InterruptedException {
     getQueue().put(job);
   }
 
   @Override
-  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit) {
+  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
+      throws InterruptedException {
     return getQueue().offer(job, timeout, unit);
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
index 9888340..834eee0 100644
--- a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
@@ -17,11 +17,6 @@
 package ai.preferred.venom.job;
 
 
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import javax.annotation.Nonnull;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -38,35 +33,21 @@
  */
 public class PriorityQueueScheduler extends AbstractQueueScheduler {
 
-  /**
-   * Logger.
-   */
-  private static final Logger LOGGER = LoggerFactory.getLogger(PriorityQueueScheduler.class);
-
   /**
    * The queue used for this scheduler.
    */
-  private final PriorityBlockingQueue<Job> queue = new PriorityBlockingQueue<>();
-
-  @Override
-  final PriorityBlockingQueue<Job> getQueue() {
-    return queue;
-  }
-
-  @Override
-  public final void add(final Request r, final Handler h, final Priority p, final Priority pf) {
-    final Job job = new BasicJob(r, h, p, pf, queue);
-    getQueue().add(job);
-    LOGGER.debug("Job {} - {} added to queue.", job.toString(), r.getUrl());
+  public PriorityQueueScheduler() {
+    super(new PriorityBlockingQueue<>());
   }
 
   @Override
-  public final void put(final @Nonnull Job job) {
+  public final void put(final @Nonnull Job job) throws InterruptedException {
     getQueue().put(job);
   }
 
   @Override
-  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit) {
+  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
+      throws InterruptedException {
     return getQueue().offer(job, timeout, unit);
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/Scheduler.java b/src/main/java/ai/preferred/venom/job/Scheduler.java
index 4a82ca1..ea82be7 100644
--- a/src/main/java/ai/preferred/venom/job/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/Scheduler.java
@@ -17,98 +17,24 @@
 package ai.preferred.venom.job;
 
 
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-
-import javax.validation.constraints.NotNull;
+import java.util.concurrent.BlockingQueue;
 
 /**
  * This interface represents only the most basic of a scheduler.
  * It imposes no restrictions or particular details on the the
  * type of queue, and allows for different future types to be returned.
  *
+ * @param <T> the type of Job to schedule
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public interface Scheduler {
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by the handler specified, and
-   * its priority can be downgraded to a minimum priority specified.
-   * </p>
-   *
-   * @param r  request to fetch when dequeued
-   * @param h  handler to be used to parse the request
-   * @param p  initial priority of the request
-   * @param pf the minimum (floor) priority of this request
-   */
-  void add(@NotNull Request r, @NotNull Handler h, Priority p, Priority pf);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by the handler specified, and
-   * its priority can be downgraded to the default minimum priority.
-   * </p>
-   *
-   * @param r request to fetch when dequeued
-   * @param h handler to be used to parse the request
-   * @param p initial priority of the request
-   */
-  void add(@NotNull Request r, @NotNull Handler h, Priority p);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by the handler specified, and
-   * it's initialised with default priority that can be downgraded to
-   * the default minimum priority.
-   * </p>
-   *
-   * @param r request to fetch when dequeued
-   * @param h handler to be used to parse the request
-   */
-  void add(@NotNull Request r, @NotNull Handler h);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by a handler defined in Router
-   * or otherwise, and its priority can be downgraded to a minimum
-   * priority specified.
-   * </p>
-   *
-   * @param r  request to fetch when dequeued
-   * @param p  initial priority of the request
-   * @param pf the minimum (floor) priority of this request
-   */
-  void add(@NotNull Request r, Priority p, Priority pf);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by a handler defined in Router
-   * or otherwise defined, and its priority can be downgraded to the
-   * default minimum priority.
-   * </p>
-   *
-   * @param r request to fetch when dequeued
-   * @param p initial priority of the request
-   */
-  void add(@NotNull Request r, Priority p);
+public interface Scheduler<T extends Job> extends BlockingQueue<T> {
 
   /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by a handler defined in Router
-   * or otherwise defined, and it's initialised with default priority
-   * that can be downgraded to the default minimum priority.
-   * </p>
+   * Get the scheduler to add jobs.
    *
-   * @param r request to fetch when dequeued
+   * @return an instance of Scheduler
    */
-  void add(@NotNull Request r);
+  ai.preferred.venom.Scheduler getScheduler();
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java b/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java
index 484a567..40bf2f9 100644
--- a/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java
@@ -30,7 +30,7 @@ public void testAddRequest() {
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
 
-    scheduler.add(vRequest);
+    scheduler.getScheduler().add(vRequest);
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
@@ -49,7 +49,7 @@ public void testAddRequestHandler() {
 
     };
 
-    scheduler.add(vRequest, handler);
+    scheduler.getScheduler().add(vRequest, handler);
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
@@ -65,10 +65,10 @@ public void testFIFOQueue() {
     final VRequest vRequest = new VRequest(url);
     final VRequest vRequestNeg = new VRequest(url);
 
-    scheduler.add(vRequest, Priority.HIGH);
-    scheduler.add(vRequestNeg, Priority.HIGHEST);
-    scheduler.add(vRequestNeg, Priority.DEFAULT);
-    scheduler.add(vRequestNeg, Priority.LOW);
+    scheduler.getScheduler().add(vRequest, Priority.HIGH);
+    scheduler.getScheduler().add(vRequestNeg, Priority.HIGHEST);
+    scheduler.getScheduler().add(vRequestNeg, Priority.DEFAULT);
+    scheduler.getScheduler().add(vRequestNeg, Priority.LOW);
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
diff --git a/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java
index 9d3a899..b818df4 100644
--- a/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java
@@ -72,7 +72,7 @@ public void testLazyQueue() {
 
     final LazyScheduler scheduler = new LazyScheduler(requests.iterator(), handler);
     final VRequest vRequest = new VRequest(url);
-    scheduler.add(vRequest);
+    scheduler.getScheduler().add(vRequest);
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
diff --git a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
index 0624b8a..f4e034c 100644
--- a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
@@ -30,7 +30,7 @@ public void testAddRequest() {
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
 
-    scheduler.add(vRequest);
+    scheduler.getScheduler().add(vRequest);
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
@@ -49,7 +49,7 @@ public void testAddRequestHandler() {
 
     };
 
-    scheduler.add(vRequest, handler);
+    scheduler.getScheduler().add(vRequest, handler);
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
@@ -65,10 +65,10 @@ public void testPriority() {
     final VRequest vRequest = new VRequest(url);
     final VRequest vRequestNeg = new VRequest(url);
 
-    scheduler.add(vRequestNeg, Priority.HIGH);
-    scheduler.add(vRequest, Priority.HIGHEST);
-    scheduler.add(vRequestNeg, Priority.DEFAULT);
-    scheduler.add(vRequestNeg, Priority.LOW);
+    scheduler.getScheduler().add(vRequestNeg, Priority.HIGH);
+    scheduler.getScheduler().add(vRequest, Priority.HIGHEST);
+    scheduler.getScheduler().add(vRequestNeg, Priority.DEFAULT);
+    scheduler.getScheduler().add(vRequestNeg, Priority.LOW);
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
@@ -84,7 +84,7 @@ public void testPriorityFloor() {
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
 
-    scheduler.add(vRequest, Priority.HIGH, Priority.NORMAL);
+    scheduler.getScheduler().add(vRequest, Priority.HIGH, Priority.NORMAL);
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);

From 2a1762550f27d3426be7dd721e388c83d88a49e8 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 5 Mar 2019 14:16:56 +0800
Subject: [PATCH 02/17] Remove excess code in ProxyProvider

---
 .../ai/preferred/venom/ProxyProvider.java     | 59 +------------------
 .../venom/job/AbstractQueueScheduler.java     |  2 +-
 .../ai/preferred/venom/FakeProxyProvider.java | 23 --------
 3 files changed, 2 insertions(+), 82 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/ProxyProvider.java b/src/main/java/ai/preferred/venom/ProxyProvider.java
index 761b03b..c2a9b43 100644
--- a/src/main/java/ai/preferred/venom/ProxyProvider.java
+++ b/src/main/java/ai/preferred/venom/ProxyProvider.java
@@ -21,9 +21,6 @@
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 
 /**
  * This interface allows the user to define proxies to be used for requests.
@@ -37,61 +34,7 @@ public interface ProxyProvider {
   /**
    * An instance of proxy provider without any proxies.
    */
-  ProxyProvider EMPTY_PROXY_PROVIDER = new ProxyProvider() {
-    @Override
-    public List<HttpHost> getProxyList() {
-      return Collections.emptyList();
-    }
-
-    @Override
-    public void add(final HttpHost proxy) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void addAll(final Collection<HttpHost> proxies) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void remove(final HttpHost proxy) {
-      throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public HttpHost get(final Request request) {
-      return null;
-    }
-  };
-
-  /**
-   * Returns a list of all proxies.
-   *
-   * @return list of proxies
-   */
-  @NotNull
-  List<HttpHost> getProxyList();
-
-  /**
-   * Add a proxy to the list.
-   *
-   * @param proxy the proxy to be added
-   */
-  void add(@NotNull HttpHost proxy);
-
-  /**
-   * Add a list of proxies to the list.
-   *
-   * @param proxies the list of proxies to be added
-   */
-  void addAll(@NotNull Collection<HttpHost> proxies);
-
-  /**
-   * Remove a proxy from the list.
-   *
-   * @param proxy the proxy to be removed
-   */
-  void remove(@NotNull HttpHost proxy);
+  ProxyProvider EMPTY_PROXY_PROVIDER = request -> null;
 
   /**
    * Returns the get proxy from the list.
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index a9c2540..48214ec 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -125,7 +125,7 @@ public static class ExternalScheduler implements ai.preferred.venom.Scheduler {
     private final BlockingQueue<Job> queue;
 
     /**
-     * Constructs an instance of ExternalScheduler
+     * Constructs an instance of ExternalScheduler.
      *
      * @param queue an instance of BlockingQueue
      */
diff --git a/src/test/java/ai/preferred/venom/FakeProxyProvider.java b/src/test/java/ai/preferred/venom/FakeProxyProvider.java
index b2f75e8..0c8d578 100644
--- a/src/test/java/ai/preferred/venom/FakeProxyProvider.java
+++ b/src/test/java/ai/preferred/venom/FakeProxyProvider.java
@@ -21,34 +21,11 @@
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 
 public class FakeProxyProvider implements ProxyProvider {
 
   private final HttpHost proxy = new HttpHost("http://127.0.0.1:8080");
 
-  @Override
-  public @NotNull List<HttpHost> getProxyList() {
-    return Collections.singletonList(proxy);
-  }
-
-  @Override
-  public void add(@NotNull HttpHost proxy) {
-
-  }
-
-  @Override
-  public void addAll(@NotNull Collection<HttpHost> proxies) {
-
-  }
-
-  @Override
-  public void remove(@NotNull HttpHost proxy) {
-
-  }
-
   @Nullable
   @Override
   public HttpHost get(@NotNull Request request) {

From 171dc568baf6715d25d2e99e387e1d56d9d12ba9 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 5 Mar 2019 14:40:57 +0800
Subject: [PATCH 03/17] Refactor job/Scheduler to QueueScheduler

---
 src/main/java/ai/preferred/venom/Crawler.java | 28 +++++++++----------
 .../ai/preferred/venom/HandlerRouter.java     |  2 +-
 .../ai/preferred/venom/ValidatorRouter.java   |  2 +-
 .../venom/job/AbstractQueueScheduler.java     |  7 +++--
 ...Scheduler.java => FIFOQueueScheduler.java} |  6 ++--
 ...Scheduler.java => LazyQueueScheduler.java} |  6 ++--
 .../{Scheduler.java => QueueScheduler.java}   |  6 ++--
 .../preferred/venom/response/VResponse.java   |  8 +++---
 .../java/ai/preferred/venom/CrawlerTest.java  | 26 ++++++++---------
 ...rTest.java => FIFOQueueSchedulerTest.java} |  8 +++---
 ...rTest.java => LazyQueueSchedulerTest.java} |  6 ++--
 11 files changed, 54 insertions(+), 51 deletions(-)
 rename src/main/java/ai/preferred/venom/job/{FIFOScheduler.java => FIFOQueueScheduler.java} (91%)
 rename src/main/java/ai/preferred/venom/job/{LazyScheduler.java => LazyQueueScheduler.java} (93%)
 rename src/main/java/ai/preferred/venom/job/{Scheduler.java => QueueScheduler.java} (88%)
 rename src/test/java/ai/preferred/venom/job/{FIFOSchedulerTest.java => FIFOQueueSchedulerTest.java} (91%)
 rename src/test/java/ai/preferred/venom/job/{LazySchedulerTest.java => LazyQueueSchedulerTest.java} (91%)

diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java
index 1399b55..fa08e4f 100644
--- a/src/main/java/ai/preferred/venom/Crawler.java
+++ b/src/main/java/ai/preferred/venom/Crawler.java
@@ -17,9 +17,9 @@
 package ai.preferred.venom;
 
 import ai.preferred.venom.fetcher.*;
+import ai.preferred.venom.job.QueueScheduler;
 import ai.preferred.venom.job.Job;
 import ai.preferred.venom.job.PriorityQueueScheduler;
-import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.CrawlerRequest;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.Response;
@@ -88,7 +88,7 @@ public final class Crawler implements Interruptible {
    * The scheduler used.
    */
   @NotNull
-  private final Scheduler<? extends Job> scheduler;
+  private final QueueScheduler<? extends Job> queueScheduler;
 
   /**
    * The maximum number of simultaneous connections.
@@ -143,7 +143,7 @@ private Crawler(final Builder builder) {
     maxTries = builder.maxTries;
     propRetainProxy = builder.propRetainProxy;
     router = builder.router;
-    scheduler = builder.scheduler;
+    queueScheduler = builder.queueScheduler;
     connections = new Semaphore(builder.maxConnections);
     session = builder.session;
     sleepScheduler = builder.sleepScheduler;
@@ -242,7 +242,7 @@ private void run() {
     long lastRequestTime = 0;
     while (!Thread.currentThread().isInterrupted() && !threadPool.isShutdown() && handlerExceptions.isEmpty()) {
       try {
-        final Job job = scheduler.poll(100, TimeUnit.MILLISECONDS);
+        final Job job = queueScheduler.poll(100, TimeUnit.MILLISECONDS);
         if (job == null) {
           if (pendingJobs.size() != 0) {
             continue;
@@ -250,7 +250,7 @@ private void run() {
           // This should only run if pendingJob == 0 && job == null
           synchronized (pendingJobs) {
             LOGGER.debug("({}) Checking for exit conditions.", crawlerThread.getName());
-            if (scheduler.peek() == null && pendingJobs.size() == 0 && exitWhenDone.get()) {
+            if (queueScheduler.peek() == null && pendingJobs.size() == 0 && exitWhenDone.get()) {
               break;
             }
           }
@@ -292,8 +292,8 @@ private void run() {
    *
    * @return the instance of scheduler used.
    */
-  public ai.preferred.venom.Scheduler getScheduler() {
-    return scheduler.getScheduler();
+  public Scheduler getScheduler() {
+    return queueScheduler.getScheduler();
   }
 
   /**
@@ -466,7 +466,7 @@ public static final class Builder {
     /**
      * The scheduler used.
      */
-    private Scheduler<? extends Job> scheduler;
+    private QueueScheduler<? extends Job> queueScheduler;
 
     /**
      * The sleep scheduler used.
@@ -490,7 +490,7 @@ private Builder() {
       workerManager = null;
       propRetainProxy = 0.05;
       router = null;
-      scheduler = new PriorityQueueScheduler();
+      queueScheduler = new PriorityQueueScheduler();
       sleepScheduler = new SleepScheduler(250, 2000);
       session = Session.EMPTY_SESSION;
     }
@@ -546,11 +546,11 @@ public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
     /**
      * Sets the Scheduler to be used, if not set, default will be chosen.
      *
-     * @param scheduler scheduler to be used.
+     * @param queueScheduler scheduler to be used.
      * @return this
      */
-    public Builder setScheduler(final @NotNull Scheduler<? extends Job> scheduler) {
-      this.scheduler = scheduler;
+    public Builder setScheduler(final @NotNull QueueScheduler<? extends Job> queueScheduler) {
+      this.queueScheduler = queueScheduler;
       return this;
     }
 
@@ -673,12 +673,12 @@ public void completed(final Request request, final Response response) {
       crawler.threadPool.execute(() -> {
         try {
           if (job.getHandler() != null) {
-            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.queueScheduler.getScheduler(),
                 crawler.session, crawler.workerManager.getWorker());
           } else if (crawler.router != null) {
             final Handler routedHandler = crawler.router.getHandler(job.getRequest());
             if (routedHandler != null) {
-              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.queueScheduler.getScheduler(),
                   crawler.session, crawler.workerManager.getWorker());
             }
           } else {
diff --git a/src/main/java/ai/preferred/venom/HandlerRouter.java b/src/main/java/ai/preferred/venom/HandlerRouter.java
index f8d445b..cfa430e 100644
--- a/src/main/java/ai/preferred/venom/HandlerRouter.java
+++ b/src/main/java/ai/preferred/venom/HandlerRouter.java
@@ -22,7 +22,7 @@
 import javax.validation.constraints.NotNull;
 
 /**
- * This interface allows the user to map request to handleable.
+ * This interface allows the user to map request to handler.
  *
  * @author Maksim Tkachenko
  */
diff --git a/src/main/java/ai/preferred/venom/ValidatorRouter.java b/src/main/java/ai/preferred/venom/ValidatorRouter.java
index e875d45..0ca0e95 100644
--- a/src/main/java/ai/preferred/venom/ValidatorRouter.java
+++ b/src/main/java/ai/preferred/venom/ValidatorRouter.java
@@ -22,7 +22,7 @@
 import javax.validation.constraints.NotNull;
 
 /**
- * This interface allows the user to map request to handleable.
+ * This interface allows the user to map request to handler.
  *
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 48214ec..21bb6cf 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -17,6 +17,7 @@
 package ai.preferred.venom.job;
 
 import ai.preferred.venom.Handler;
+import ai.preferred.venom.Scheduler;
 import ai.preferred.venom.request.Request;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -31,7 +32,7 @@
  * @author Ween Jiann Lee
  * @author Maksim Tkachenko
  */
-public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements Scheduler<Job> {
+public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements QueueScheduler<Job> {
 
   /**
    * The queue used for this scheduler.
@@ -41,7 +42,7 @@ public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implemen
   /**
    * The adding part of the scheduler.
    */
-  private final ai.preferred.venom.Scheduler scheduler;
+  private final Scheduler scheduler;
 
   /**
    * Constructs an instance of AbstractQueueScheduler.
@@ -112,7 +113,7 @@ protected final BlockingQueue<Job> getQueue() {
   /**
    * An implementation of ai.preferred.venom.Scheduler using BasicJob.
    */
-  public static class ExternalScheduler implements ai.preferred.venom.Scheduler {
+  public static class ExternalScheduler implements Scheduler {
 
     /**
      * Logger.
diff --git a/src/main/java/ai/preferred/venom/job/FIFOScheduler.java b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
similarity index 91%
rename from src/main/java/ai/preferred/venom/job/FIFOScheduler.java
rename to src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
index 36f2c5f..6876bc4 100644
--- a/src/main/java/ai/preferred/venom/job/FIFOScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
@@ -29,12 +29,12 @@
  *
  * @author Ween Jiann Lee
  */
-public class FIFOScheduler extends AbstractQueueScheduler {
+public class FIFOQueueScheduler extends AbstractQueueScheduler {
 
   /**
-   * Constructs an instance of FIFOScheduler.
+   * Constructs an instance of FIFOQueueScheduler.
    */
-  public FIFOScheduler() {
+  public FIFOQueueScheduler() {
     super(new LinkedBlockingQueue<>());
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/LazyScheduler.java b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
similarity index 93%
rename from src/main/java/ai/preferred/venom/job/LazyScheduler.java
rename to src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
index ae25c36..a697c50 100644
--- a/src/main/java/ai/preferred/venom/job/LazyScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
@@ -36,7 +36,7 @@
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public class LazyScheduler extends AbstractQueueScheduler {
+public class LazyQueueScheduler extends AbstractQueueScheduler {
 
   /**
    * An object to synchronise upon.
@@ -59,7 +59,7 @@ public class LazyScheduler extends AbstractQueueScheduler {
    * @param requests An iterator to obtain requests
    * @param handler  The default handler to use
    */
-  public LazyScheduler(final Iterator<Request> requests, final Handler handler) {
+  public LazyQueueScheduler(final Iterator<Request> requests, final Handler handler) {
     super(new PriorityBlockingQueue<>());
     this.requests = requests;
     this.handler = handler;
@@ -70,7 +70,7 @@ public LazyScheduler(final Iterator<Request> requests, final Handler handler) {
    *
    * @param requests An iterator to obtain requests
    */
-  public LazyScheduler(final Iterator<Request> requests) {
+  public LazyQueueScheduler(final Iterator<Request> requests) {
     this(requests, null);
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/Scheduler.java b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
similarity index 88%
rename from src/main/java/ai/preferred/venom/job/Scheduler.java
rename to src/main/java/ai/preferred/venom/job/QueueScheduler.java
index ea82be7..81aa02e 100644
--- a/src/main/java/ai/preferred/venom/job/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
@@ -17,6 +17,8 @@
 package ai.preferred.venom.job;
 
 
+import ai.preferred.venom.Scheduler;
+
 import java.util.concurrent.BlockingQueue;
 
 /**
@@ -28,13 +30,13 @@
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public interface Scheduler<T extends Job> extends BlockingQueue<T> {
+public interface QueueScheduler<T extends Job> extends BlockingQueue<T> {
 
   /**
    * Get the scheduler to add jobs.
    *
    * @return an instance of Scheduler
    */
-  ai.preferred.venom.Scheduler getScheduler();
+  Scheduler getScheduler();
 
 }
diff --git a/src/main/java/ai/preferred/venom/response/VResponse.java b/src/main/java/ai/preferred/venom/response/VResponse.java
index 0e3a92d..8b013d3 100644
--- a/src/main/java/ai/preferred/venom/response/VResponse.java
+++ b/src/main/java/ai/preferred/venom/response/VResponse.java
@@ -105,19 +105,19 @@ public final String getHtml(final Charset charset) {
   }
 
   /**
-   * Returns a Joup document of this response.
+   * Returns a jsoup document of this response.
    *
-   * @return Jsoup document of response
+   * @return jsoup document of response
    */
   public final Document getJsoup() {
     return Jsoup.parse(getHtml(), getBaseUrl());
   }
 
   /**
-   * Returns a Jsoup document of this response.
+   * Returns a jsoup document of this response.
    *
    * @param charset use specified charset for this html document
-   * @return Jsoup document of response
+   * @return jsoup document of response
    */
   public final Document getJsoup(final Charset charset) {
     return Jsoup.parse(getHtml(charset), getBaseUrl());
diff --git a/src/test/java/ai/preferred/venom/CrawlerTest.java b/src/test/java/ai/preferred/venom/CrawlerTest.java
index a41aa80..d66ced0 100644
--- a/src/test/java/ai/preferred/venom/CrawlerTest.java
+++ b/src/test/java/ai/preferred/venom/CrawlerTest.java
@@ -18,8 +18,8 @@
 
 import ai.preferred.venom.fetcher.FakeFetcher;
 import ai.preferred.venom.fetcher.Fetcher;
-import ai.preferred.venom.job.FIFOScheduler;
-import ai.preferred.venom.job.LazyScheduler;
+import ai.preferred.venom.job.FIFOQueueScheduler;
+import ai.preferred.venom.job.LazyQueueScheduler;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.request.VRequest;
 import org.apache.http.HttpHost;
@@ -59,7 +59,7 @@ public void testCrawler() throws Exception {
         .setFetcher(fetcher)
         .setMaxConnections(1)
         .setMaxTries(2)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -85,7 +85,7 @@ public void testCrawlerStartAndClose() throws Exception {
         .setFetcher(fetcher)
         .setMaxConnections(1)
         .setMaxTries(2)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build();
 
@@ -113,7 +113,7 @@ public void testRetry() throws Exception {
         .setFetcher(fetcher)
         .setMaxConnections(1)
         .setMaxTries(5)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -141,7 +141,7 @@ public void testMaxTries() throws Exception {
         .setFetcher(fetcher)
         .setMaxConnections(1)
         .setMaxTries(5)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -174,7 +174,7 @@ public void testProxyProportionRemoved() throws Exception {
         .setMaxConnections(1)
         .setPropRetainProxy(0.2)
         .setMaxTries(5)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -207,7 +207,7 @@ public void testProxyProportionRetained() throws Exception {
         .setMaxConnections(1)
         .setPropRetainProxy(0.2)
         .setMaxTries(5)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -239,7 +239,7 @@ public void testLazySchedulerIntegration() throws Exception {
         .setMaxConnections(1)
         .setPropRetainProxy(0.2)
         .setMaxTries(5)
-        .setScheduler(new LazyScheduler(requests.iterator(), handler))
+        .setScheduler(new LazyQueueScheduler(requests.iterator(), handler))
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
@@ -261,7 +261,7 @@ public void testUrlRouterIntegration() throws Exception {
     try (final Crawler crawler = Crawler.builder()
         .setFetcher(fetcher)
         .setMaxTries(1)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .setHandlerRouter(urlRouter)
         .build()
@@ -285,7 +285,7 @@ public void testFatalHandlerException() {
           .setFetcher(fetcher)
           .setMaxTries(1)
           .setMaxConnections(1)
-          .setScheduler(new FIFOScheduler())
+          .setScheduler(new FIFOQueueScheduler())
           .setSleepScheduler(new SleepScheduler(0))
           .build()
           .start()) {
@@ -324,7 +324,7 @@ public void testSessionIntegration() throws Exception {
     try (final Crawler crawler = Crawler.builder()
         .setFetcher(fetcher)
         .setMaxTries(1)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSleepScheduler(new SleepScheduler(0))
         .setSession(session)
         .build()
@@ -355,7 +355,7 @@ public void testInterruptAndClose() throws Exception {
     final Crawler crawler = Crawler.builder()
         .setFetcher(fetcher)
         .setMaxTries(1)
-        .setScheduler(new FIFOScheduler())
+        .setScheduler(new FIFOQueueScheduler())
         .setSession(session)
         .build()
         .start();
diff --git a/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
similarity index 91%
rename from src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
index 40bf2f9..e4cfdfa 100644
--- a/src/test/java/ai/preferred/venom/job/FIFOSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
@@ -21,11 +21,11 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
-public class FIFOSchedulerTest {
+public class FIFOQueueSchedulerTest {
 
   @Test
   public void testAddRequest() {
-    final FIFOScheduler scheduler = new FIFOScheduler();
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
 
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
@@ -40,7 +40,7 @@ public void testAddRequest() {
 
   @Test
   public void testAddRequestHandler() {
-    final FIFOScheduler scheduler = new FIFOScheduler();
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
 
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
@@ -59,7 +59,7 @@ public void testAddRequestHandler() {
 
   @Test
   public void testFIFOQueue() {
-    final FIFOScheduler scheduler = new FIFOScheduler();
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
 
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
diff --git a/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
similarity index 91%
rename from src/test/java/ai/preferred/venom/job/LazySchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
index b818df4..16fc93f 100644
--- a/src/test/java/ai/preferred/venom/job/LazySchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
@@ -25,7 +25,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class LazySchedulerTest {
+public class LazyQueueSchedulerTest {
 
   @Test
   public void testIterator() {
@@ -44,7 +44,7 @@ public void testIterator() {
 
     };
 
-    final LazyScheduler scheduler = new LazyScheduler(requests.iterator(), handler);
+    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
@@ -70,7 +70,7 @@ public void testLazyQueue() {
     };
 
 
-    final LazyScheduler scheduler = new LazyScheduler(requests.iterator(), handler);
+    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
     final VRequest vRequest = new VRequest(url);
     scheduler.getScheduler().add(vRequest);
 

From 3390726cb378d58940ea8d6039480fd6bae4a8f7 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 5 Mar 2019 15:09:08 +0800
Subject: [PATCH 04/17] Updated UrlRouter to threadsafe

---
 .../java/ai/preferred/venom/UrlRouter.java    | 47 +++++++++++++++----
 1 file changed, 39 insertions(+), 8 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/UrlRouter.java b/src/main/java/ai/preferred/venom/UrlRouter.java
index 675b285..bfc6627 100644
--- a/src/main/java/ai/preferred/venom/UrlRouter.java
+++ b/src/main/java/ai/preferred/venom/UrlRouter.java
@@ -21,6 +21,7 @@
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Pattern;
 
 /**
@@ -47,6 +48,16 @@ public class UrlRouter implements HandlerRouter, ValidatorRouter {
    */
   private final Map<Pattern, Validator> validatorRules = new LinkedHashMap<>();
 
+  /**
+   * A read write lock for handler rules.
+   */
+  private final ReentrantReadWriteLock handlerRulesLock = new ReentrantReadWriteLock();
+
+  /**
+   * A read write lock for validator rules.
+   */
+  private final ReentrantReadWriteLock validatorRulesLock = new ReentrantReadWriteLock();
+
   /**
    * Constructs a url router without default handler.
    */
@@ -74,7 +85,12 @@ public UrlRouter(final Handler defaultHandler) {
    * @return this.
    */
   public final UrlRouter register(final Pattern urlPattern, final Handler handler) {
-    handlerRules.put(urlPattern, handler);
+    handlerRulesLock.writeLock().lock();
+    try {
+      handlerRules.put(urlPattern, handler);
+    } finally {
+      handlerRulesLock.writeLock().unlock();
+    }
     return this;
   }
 
@@ -89,7 +105,12 @@ public final UrlRouter register(final Pattern urlPattern, final Handler handler)
    * @return this.
    */
   public final UrlRouter register(final Pattern urlPattern, final Validator validator) {
-    validatorRules.put(urlPattern, validator);
+    validatorRulesLock.writeLock().lock();
+    try {
+      validatorRules.put(urlPattern, validator);
+    } finally {
+      validatorRulesLock.writeLock().unlock();
+    }
     return this;
   }
 
@@ -112,10 +133,15 @@ public final UrlRouter register(final Pattern urlPattern, final Handler handler,
 
   @Override
   public final Handler getHandler(final Request request) {
-    for (final Map.Entry<Pattern, Handler> rule : handlerRules.entrySet()) {
-      if (rule.getKey().matcher(request.getUrl()).matches()) {
-        return rule.getValue();
+    handlerRulesLock.readLock().lock();
+    try {
+      for (final Map.Entry<Pattern, Handler> rule : handlerRules.entrySet()) {
+        if (rule.getKey().matcher(request.getUrl()).matches()) {
+          return rule.getValue();
+        }
       }
+    } finally {
+      handlerRulesLock.readLock().unlock();
     }
 
     if (defaultHandler != null) {
@@ -127,10 +153,15 @@ public final Handler getHandler(final Request request) {
 
   @Override
   public final Validator getValidator(final Request request) {
-    for (final Map.Entry<Pattern, Validator> rule : validatorRules.entrySet()) {
-      if (rule.getKey().matcher(request.getUrl()).matches()) {
-        return rule.getValue();
+    validatorRulesLock.readLock().lock();
+    try {
+      for (final Map.Entry<Pattern, Validator> rule : validatorRules.entrySet()) {
+        if (rule.getKey().matcher(request.getUrl()).matches()) {
+          return rule.getValue();
+        }
       }
+    } finally {
+      validatorRulesLock.readLock().unlock();
     }
 
     return Validator.ALWAYS_VALID;

From 15d20eab154a42f77140cfb53c60bcadf758db67 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 5 Mar 2019 18:02:29 +0800
Subject: [PATCH 05/17] Fixed CrawlerTest

---
 src/test/java/ai/preferred/venom/CrawlerTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/ai/preferred/venom/CrawlerTest.java b/src/test/java/ai/preferred/venom/CrawlerTest.java
index d66ced0..1ac9a22 100644
--- a/src/test/java/ai/preferred/venom/CrawlerTest.java
+++ b/src/test/java/ai/preferred/venom/CrawlerTest.java
@@ -360,8 +360,8 @@ public void testInterruptAndClose() throws Exception {
         .build()
         .start();
 
-    crawler.getScheduler().add(vRequest, assertHandler);
     crawler.interruptAndClose();
+    crawler.getScheduler().add(vRequest, assertHandler);
 
     Assertions.assertEquals(0, fetcher.getCounter());
   }

From 3e803c001f607c1955096c6212eaaf738203ee4a Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Wed, 6 Mar 2019 10:13:31 +0800
Subject: [PATCH 06/17] Minor refactoring

---
 src/main/java/ai/preferred/venom/Crawler.java  |  1 +
 .../preferred/venom/FatalHandlerException.java | 18 +++++++++++++++++-
 src/main/java/ai/preferred/venom/Handler.java  |  1 +
 .../venom/job/AbstractQueueScheduler.java      |  5 ++---
 .../ai/preferred/venom/job/QueueScheduler.java |  2 --
 .../preferred/venom/{ => job}/Scheduler.java   |  4 ++--
 6 files changed, 23 insertions(+), 8 deletions(-)
 rename src/main/java/ai/preferred/venom/{ => job}/Scheduler.java (97%)

diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java
index fa08e4f..17c3a4e 100644
--- a/src/main/java/ai/preferred/venom/Crawler.java
+++ b/src/main/java/ai/preferred/venom/Crawler.java
@@ -20,6 +20,7 @@
 import ai.preferred.venom.job.QueueScheduler;
 import ai.preferred.venom.job.Job;
 import ai.preferred.venom.job.PriorityQueueScheduler;
+import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.CrawlerRequest;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.Response;
diff --git a/src/main/java/ai/preferred/venom/FatalHandlerException.java b/src/main/java/ai/preferred/venom/FatalHandlerException.java
index b89ef9d..ee3375c 100644
--- a/src/main/java/ai/preferred/venom/FatalHandlerException.java
+++ b/src/main/java/ai/preferred/venom/FatalHandlerException.java
@@ -1,9 +1,25 @@
+/*
+ * Copyright 2018 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package ai.preferred.venom;
 
 /**
  * This class defines fatal runtime exception for {@link Handler}.
  * If {@link Handler#handle(ai.preferred.venom.request.Request,
- * ai.preferred.venom.response.VResponse, ai.preferred.venom.Scheduler,
+ * ai.preferred.venom.response.VResponse, ai.preferred.venom.job.Scheduler,
  * Session, Worker)} encounters unexpected situation, it can signal
  * {@link Crawler} to stop execution by throwing this exception.
  *
diff --git a/src/main/java/ai/preferred/venom/Handler.java b/src/main/java/ai/preferred/venom/Handler.java
index 200be4b..e7e6702 100644
--- a/src/main/java/ai/preferred/venom/Handler.java
+++ b/src/main/java/ai/preferred/venom/Handler.java
@@ -16,6 +16,7 @@
 
 package ai.preferred.venom;
 
+import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.VResponse;
 
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 21bb6cf..2dfa7de 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -17,7 +17,6 @@
 package ai.preferred.venom.job;
 
 import ai.preferred.venom.Handler;
-import ai.preferred.venom.Scheduler;
 import ai.preferred.venom.request.Request;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,7 +54,7 @@ protected AbstractQueueScheduler(final BlockingQueue<Job> queue) {
   }
 
   @Override
-  public final ai.preferred.venom.Scheduler getScheduler() {
+  public final Scheduler getScheduler() {
     return scheduler;
   }
 
@@ -111,7 +110,7 @@ protected final BlockingQueue<Job> getQueue() {
   }
 
   /**
-   * An implementation of ai.preferred.venom.Scheduler using BasicJob.
+   * An implementation of ai.preferred.venom.job.Scheduler using BasicJob.
    */
   public static class ExternalScheduler implements Scheduler {
 
diff --git a/src/main/java/ai/preferred/venom/job/QueueScheduler.java b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
index 81aa02e..ac7cd25 100644
--- a/src/main/java/ai/preferred/venom/job/QueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
@@ -17,8 +17,6 @@
 package ai.preferred.venom.job;
 
 
-import ai.preferred.venom.Scheduler;
-
 import java.util.concurrent.BlockingQueue;
 
 /**
diff --git a/src/main/java/ai/preferred/venom/Scheduler.java b/src/main/java/ai/preferred/venom/job/Scheduler.java
similarity index 97%
rename from src/main/java/ai/preferred/venom/Scheduler.java
rename to src/main/java/ai/preferred/venom/job/Scheduler.java
index 46f437b..2c812d8 100644
--- a/src/main/java/ai/preferred/venom/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/Scheduler.java
@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-package ai.preferred.venom;
+package ai.preferred.venom.job;
 
-import ai.preferred.venom.job.Priority;
+import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.Request;
 
 import javax.validation.constraints.NotNull;

From 634b37dc3072810d18c02337e342c787a5b4106c Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Wed, 6 Mar 2019 15:26:59 +0800
Subject: [PATCH 07/17] Minor refactoring

---
 src/main/java/ai/preferred/venom/ProxyProvider.java    |  5 -----
 .../ai/preferred/venom/job/AbstractQueueScheduler.java | 10 +++++-----
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/ProxyProvider.java b/src/main/java/ai/preferred/venom/ProxyProvider.java
index c2a9b43..8c34538 100644
--- a/src/main/java/ai/preferred/venom/ProxyProvider.java
+++ b/src/main/java/ai/preferred/venom/ProxyProvider.java
@@ -31,11 +31,6 @@
  */
 public interface ProxyProvider {
 
-  /**
-   * An instance of proxy provider without any proxies.
-   */
-  ProxyProvider EMPTY_PROXY_PROVIDER = request -> null;
-
   /**
    * Returns the get proxy from the list.
    *
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 2dfa7de..9bc1905 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -50,7 +50,7 @@ public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implemen
    */
   protected AbstractQueueScheduler(final BlockingQueue<Job> queue) {
     this.queue = queue;
-    this.scheduler = new ExternalScheduler(queue);
+    this.scheduler = new JobScheduler(queue);
   }
 
   @Override
@@ -112,12 +112,12 @@ protected final BlockingQueue<Job> getQueue() {
   /**
    * An implementation of ai.preferred.venom.job.Scheduler using BasicJob.
    */
-  public static class ExternalScheduler implements Scheduler {
+  public static class JobScheduler implements Scheduler {
 
     /**
      * Logger.
      */
-    private static final Logger LOGGER = LoggerFactory.getLogger(ExternalScheduler.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduler.class);
 
     /**
      * The queue used for this scheduler.
@@ -125,11 +125,11 @@ public static class ExternalScheduler implements Scheduler {
     private final BlockingQueue<Job> queue;
 
     /**
-     * Constructs an instance of ExternalScheduler.
+     * Constructs an instance of JobScheduler.
      *
      * @param queue an instance of BlockingQueue
      */
-    public ExternalScheduler(final BlockingQueue<Job> queue) {
+    public JobScheduler(final BlockingQueue<Job> queue) {
       this.queue = queue;
     }
 

From 3134c7ad172b19eefe9b7b71fa7560a1e6ede76d Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 12 Mar 2019 10:22:56 +0800
Subject: [PATCH 08/17] Modified AsyncFetcher.Builder and Crawler.Builder to
 fail fast

---
 src/main/java/ai/preferred/venom/Crawler.java | 42 ++++++++++++++-----
 .../preferred/venom/fetcher/AsyncFetcher.java | 36 ++++++++++++++--
 .../venom/uagent/DefaultUserAgent.java        |  2 +-
 .../ai/preferred/venom/uagent/UserAgent.java  |  3 --
 4 files changed, 65 insertions(+), 18 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java
index 17c3a4e..c86e618 100644
--- a/src/main/java/ai/preferred/venom/Crawler.java
+++ b/src/main/java/ai/preferred/venom/Crawler.java
@@ -17,9 +17,9 @@
 package ai.preferred.venom;
 
 import ai.preferred.venom.fetcher.*;
-import ai.preferred.venom.job.QueueScheduler;
 import ai.preferred.venom.job.Job;
 import ai.preferred.venom.job.PriorityQueueScheduler;
+import ai.preferred.venom.job.QueueScheduler;
 import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.CrawlerRequest;
 import ai.preferred.venom.request.Request;
@@ -484,7 +484,7 @@ public static final class Builder {
      */
     private Builder() {
       fetcher = AsyncFetcher.buildDefault();
-      maxConnections = Runtime.getRuntime().availableProcessors() * 50;
+      maxConnections = 32;
       maxTries = 50;
       name = "Crawler";
       parallelism = Runtime.getRuntime().availableProcessors();
@@ -503,6 +503,9 @@ private Builder() {
      * @return this
      */
     public Builder setName(final @NotNull String name) {
+      if (name == null) {
+        throw new IllegalStateException("Attribute 'name' cannot be null.");
+      }
       this.name = name;
       return this;
     }
@@ -514,6 +517,9 @@ public Builder setName(final @NotNull String name) {
      * @return this
      */
     public Builder setFetcher(final @NotNull Fetcher fetcher) {
+      if (fetcher == null) {
+        throw new IllegalStateException("Attribute 'fetcher' cannot be null.");
+      }
       this.fetcher = fetcher;
       return this;
     }
@@ -526,10 +532,9 @@ public Builder setFetcher(final @NotNull Fetcher fetcher) {
      */
     public Builder setParallelism(final int parallelism) {
       if (parallelism <= 0) {
-        LOGGER.warn("Attribute 'numThreads' not within range, defaulting to system default.");
-      } else {
-        this.parallelism = parallelism;
+        throw new IllegalStateException("Attribute 'parallelism' must be more or equal to 1.");
       }
+      this.parallelism = parallelism;
       return this;
     }
 
@@ -540,6 +545,9 @@ public Builder setParallelism(final int parallelism) {
      * @return this
      */
     public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
+      if (workerManager == null) {
+        throw new IllegalStateException("Attribute 'workerManager' cannot be null.");
+      }
       this.workerManager = workerManager;
       return this;
     }
@@ -551,6 +559,9 @@ public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
      * @return this
      */
     public Builder setScheduler(final @NotNull QueueScheduler<? extends Job> queueScheduler) {
+      if (queueScheduler == null) {
+        throw new IllegalStateException("Attribute 'queueScheduler' cannot be null.");
+      }
       this.queueScheduler = queueScheduler;
       return this;
     }
@@ -561,7 +572,7 @@ public Builder setScheduler(final @NotNull QueueScheduler<? extends Job> queueSc
      * @param router handler router to be used.
      * @return this
      */
-    public Builder setHandlerRouter(final @NotNull HandlerRouter router) {
+    public Builder setHandlerRouter(final HandlerRouter router) {
       this.router = router;
       return this;
     }
@@ -573,6 +584,9 @@ public Builder setHandlerRouter(final @NotNull HandlerRouter router) {
      * @return this
      */
     public Builder setMaxConnections(final int maxConnections) {
+      if (maxConnections <= 0) {
+        throw new IllegalStateException("Attribute 'maxConnections' must be more or equal to 1.");
+      }
       this.maxConnections = maxConnections;
       return this;
     }
@@ -585,6 +599,9 @@ public Builder setMaxConnections(final int maxConnections) {
      * @return this
      */
     public Builder setMaxTries(final int maxTries) {
+      if (maxTries <= 0) {
+        throw new IllegalStateException("Attribute 'maxTries' must be more or equal to 1.");
+      }
       this.maxTries = maxTries;
       return this;
     }
@@ -602,10 +619,9 @@ public Builder setMaxTries(final int maxTries) {
      */
     public Builder setPropRetainProxy(final double propRetainProxy) {
       if (propRetainProxy > 1 || propRetainProxy < 0) {
-        LOGGER.warn("Attribute 'propRetainProxy' not within range, defaulting to 0.05.");
-      } else {
-        this.propRetainProxy = propRetainProxy;
+        throw new IllegalStateException("Attribute 'propRetainProxy' not within range, must be (0,1].");
       }
+      this.propRetainProxy = propRetainProxy;
       return this;
     }
 
@@ -621,12 +637,15 @@ public Builder setSleepScheduler(final SleepScheduler sleepScheduler) {
     }
 
     /**
-     * Sets the Session to be used, if not set, defaults to none.
+     * Sets the Session to be used, if not set, defaults to {@code Session.EMPTY_SESSION}.
      *
      * @param session Sessions where variables are defined
      * @return this
      */
-    public Builder setSession(final @NotNull Session session) {
+    public Builder setSession(final Session session) {
+      if (session == null) {
+        this.session = Session.EMPTY_SESSION;
+      }
       this.session = session;
       return this;
     }
@@ -704,6 +723,7 @@ public void failed(final Request request, final Exception ex) {
             || (ex instanceof ValidationException && ((ValidationException) ex).getStatus() == Validator.Status.STOP)) {
           crawler.pendingJobs.remove(job);
         } else {
+          // Synchronisation required to prevent crawler stopping incorrectly.
           synchronized (crawler.pendingJobs) {
             crawler.pendingJobs.remove(job);
             if (job.getTryCount() < crawler.maxTries) {
diff --git a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
index deac462..4ab0a4e 100644
--- a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
+++ b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
@@ -544,6 +544,9 @@ private Builder() {
      * @return this
      */
     public Builder register(final @NotNull Callback callback) {
+      if (callback == null) {
+        throw new IllegalStateException("Attribute 'callback' cannot be null.");
+      }
       this.callbacks.add(callback);
       return this;
     }
@@ -567,7 +570,7 @@ public Builder disableCookies() {
      * @param fileManager file manager to be used.
      * @return this
      */
-    public Builder setFileManager(final @NotNull FileManager fileManager) {
+    public Builder setFileManager(final FileManager fileManager) {
       this.fileManager = fileManager;
       return this;
     }
@@ -579,6 +582,9 @@ public Builder setFileManager(final @NotNull FileManager fileManager) {
      * @return this
      */
     public Builder setHeaders(final @NotNull Map<String, String> headers) {
+      if (headers == null) {
+        throw new IllegalStateException("Attribute 'headers' cannot be null.");
+      }
       this.headers = headers;
       return this;
     }
@@ -590,6 +596,9 @@ public Builder setHeaders(final @NotNull Map<String, String> headers) {
      * @return this
      */
     public Builder setNumIoThreads(final int numIoThreads) {
+      if (numIoThreads == -1 ^ numIoThreads <= 0) {
+        throw new IllegalStateException("Attribute 'numIoThreads' must be -1, or more or equal to 1.");
+      }
       this.numIoThreads = numIoThreads;
       return this;
     }
@@ -601,6 +610,9 @@ public Builder setNumIoThreads(final int numIoThreads) {
      * @return this
      */
     public Builder setMaxConnections(final int maxConnections) {
+      if (maxConnections <= 0) {
+        throw new IllegalStateException("Attribute 'maxConnections' must be more or equal to 1.");
+      }
       this.maxConnections = maxConnections;
       return this;
     }
@@ -613,6 +625,9 @@ public Builder setMaxConnections(final int maxConnections) {
      * @return this
      */
     public Builder setMaxRouteConnections(final int maxRouteConnections) {
+      if (maxRouteConnections <= 0) {
+        throw new IllegalStateException("Attribute 'maxRouteConnections' must be more or equal to 1.");
+      }
       this.maxRouteConnections = maxRouteConnections;
       return this;
     }
@@ -623,7 +638,7 @@ public Builder setMaxRouteConnections(final int maxRouteConnections) {
      * @param proxyProvider proxy provider to be used.
      * @return this
      */
-    public Builder setProxyProvider(final @NotNull ProxyProvider proxyProvider) {
+    public Builder setProxyProvider(final ProxyProvider proxyProvider) {
       this.proxyProvider = proxyProvider;
       return this;
     }
@@ -662,6 +677,9 @@ public Builder setStopCodes(final int... codes) {
      * @return this
      */
     public Builder setThreadFactory(final @NotNull ThreadFactory threadFactory) {
+      if (threadFactory == null) {
+        throw new IllegalStateException("Attribute 'threadFactory' cannot be null.");
+      }
       this.threadFactory = threadFactory;
       return this;
     }
@@ -673,6 +691,9 @@ public Builder setThreadFactory(final @NotNull ThreadFactory threadFactory) {
      * @return this
      */
     public Builder setUserAgent(final @NotNull UserAgent userAgent) {
+      if (userAgent == null) {
+        throw new IllegalStateException("Attribute 'userAgent' cannot be null.");
+      }
       this.userAgent = userAgent;
       return this;
     }
@@ -727,7 +748,7 @@ public Builder setRedirectStrategy(final RedirectStrategy redirectStrategy) {
      * @param router router validator setValidatorRouter to be used.
      * @return this
      */
-    public Builder setValidatorRouter(final @NotNull ValidatorRouter router) {
+    public Builder setValidatorRouter(final ValidatorRouter router) {
       this.router = router;
       return this;
     }
@@ -741,6 +762,9 @@ public Builder setValidatorRouter(final @NotNull ValidatorRouter router) {
      * @return this
      */
     public Builder setConnectionRequestTimeout(final int connectionRequestTimeout) {
+      if (connectionRequestTimeout == -1 ^ connectionRequestTimeout < 0) {
+        throw new IllegalStateException("Attribute 'connectTimeout' must be -1, or more or equal to 0.");
+      }
       this.connectionRequestTimeout = connectionRequestTimeout;
       return this;
     }
@@ -753,6 +777,9 @@ public Builder setConnectionRequestTimeout(final int connectionRequestTimeout) {
      * @return this
      */
     public Builder setConnectTimeout(final int connectTimeout) {
+      if (connectTimeout == -1 ^ connectTimeout < 0) {
+        throw new IllegalStateException("Attribute 'connectTimeout' must be -1, or more or equal to 0.");
+      }
       this.connectTimeout = connectTimeout;
       return this;
     }
@@ -766,6 +793,9 @@ public Builder setConnectTimeout(final int connectTimeout) {
      * @return this
      */
     public Builder setSocketTimeout(final int socketTimeout) {
+      if (socketTimeout == -1 ^ socketTimeout < 0) {
+        throw new IllegalStateException("Attribute 'socketTimeout' must be -1, or more or equal to 0.");
+      }
       this.socketTimeout = socketTimeout;
       return this;
     }
diff --git a/src/main/java/ai/preferred/venom/uagent/DefaultUserAgent.java b/src/main/java/ai/preferred/venom/uagent/DefaultUserAgent.java
index 5f9bc41..c1ffdbc 100644
--- a/src/main/java/ai/preferred/venom/uagent/DefaultUserAgent.java
+++ b/src/main/java/ai/preferred/venom/uagent/DefaultUserAgent.java
@@ -24,7 +24,7 @@ public class DefaultUserAgent implements UserAgent {
 
   @Override
   public final String get() {
-    return "Venom/4.0.1 ("
+    return "Venom/4.1 ("
         + System.getProperty("os.name") + "; "
         + System.getProperty("os.version") + "; "
         + System.getProperty("os.arch")
diff --git a/src/main/java/ai/preferred/venom/uagent/UserAgent.java b/src/main/java/ai/preferred/venom/uagent/UserAgent.java
index ac58373..1be8227 100644
--- a/src/main/java/ai/preferred/venom/uagent/UserAgent.java
+++ b/src/main/java/ai/preferred/venom/uagent/UserAgent.java
@@ -16,8 +16,6 @@
 
 package ai.preferred.venom.uagent;
 
-import javax.annotation.Nullable;
-
 /**
  * This interface allows the user to define a user agent string.
  *
@@ -30,7 +28,6 @@ public interface UserAgent {
    *
    * @return user agent
    */
-  @Nullable
   String get();
 
 }

From 1767f35d78776998d0bb15c1e68c4235cf5c166b Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Tue, 12 Mar 2019 11:04:12 +0800
Subject: [PATCH 09/17] Added tests for Crawler.Builder and
 AsyncFetcher.Builder

---
 .../ai/preferred/venom/ProxyProvider.java     |   2 -
 .../venom/ThreadedWorkerManager.java          |   7 +
 .../preferred/venom/fetcher/AsyncFetcher.java |   7 +-
 .../preferred/venom/CrawlerBuilderTest.java   |  96 +++++++++
 .../fetcher/AsyncFetcherBuilderTest.java      | 196 ++++++++++++++++++
 5 files changed, 304 insertions(+), 4 deletions(-)
 create mode 100644 src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
 create mode 100644 src/test/java/ai/preferred/venom/fetcher/AsyncFetcherBuilderTest.java

diff --git a/src/main/java/ai/preferred/venom/ProxyProvider.java b/src/main/java/ai/preferred/venom/ProxyProvider.java
index 8c34538..7ef387c 100644
--- a/src/main/java/ai/preferred/venom/ProxyProvider.java
+++ b/src/main/java/ai/preferred/venom/ProxyProvider.java
@@ -19,7 +19,6 @@
 import ai.preferred.venom.request.Request;
 import org.apache.http.HttpHost;
 
-import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
 
 /**
@@ -37,7 +36,6 @@ public interface ProxyProvider {
    * @param request the request to be made
    * @return the proxy to use
    */
-  @Nullable
   HttpHost get(@NotNull Request request);
 
 }
diff --git a/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java b/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java
index 89c02df..f92b0b8 100644
--- a/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java
+++ b/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java
@@ -44,6 +44,13 @@ public class ThreadedWorkerManager implements WorkerManager {
    */
   private final Worker worker;
 
+  /**
+   * Constructs a threaded worker manager with a specified executor.
+   */
+  public ThreadedWorkerManager() {
+    this(null);
+  }
+
   /**
    * Constructs a threaded worker manager with a specified executor.
    *
diff --git a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
index 4ab0a4e..f3ea692 100644
--- a/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
+++ b/src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java
@@ -596,8 +596,8 @@ public Builder setHeaders(final @NotNull Map<String, String> headers) {
      * @return this
      */
     public Builder setNumIoThreads(final int numIoThreads) {
-      if (numIoThreads == -1 ^ numIoThreads <= 0) {
-        throw new IllegalStateException("Attribute 'numIoThreads' must be -1, or more or equal to 1.");
+      if (numIoThreads <= 0) {
+        throw new IllegalStateException("Attribute 'numIoThreads' must be more or equal to 1.");
       }
       this.numIoThreads = numIoThreads;
       return this;
@@ -661,6 +661,9 @@ public Builder setSslContext(final SSLContext sslContext) {
      * @return this
      */
     public Builder setStopCodes(final int... codes) {
+      if (codes == null) {
+        throw new IllegalStateException("Attribute 'codes' cannot be null.");
+      }
       ImmutableSet.Builder<Integer> builder = new ImmutableSet.Builder<>();
       for (int code : codes) {
         builder.add(code);
diff --git a/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java b/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
new file mode 100644
index 0000000..7d6211f
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/CrawlerBuilderTest.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom;
+
+import ai.preferred.venom.fetcher.AsyncFetcher;
+import ai.preferred.venom.job.FIFOQueueScheduler;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CrawlerBuilderTest {
+
+  @Test
+  void testSetFetcher() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setFetcher(null));
+    Crawler.builder().setFetcher(AsyncFetcher.buildDefault());
+  }
+
+  @Test
+  void testSetMaxConnection() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setMaxConnections(0));
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setMaxConnections(-1));
+    Crawler.builder().setMaxConnections(1);
+  }
+
+  @Test
+  void testSetHandlerRouter() {
+    Crawler.builder().setHandlerRouter(null);
+    Crawler.builder().setHandlerRouter(new UrlRouter());
+  }
+
+  @Test
+  void testSetMaxTries() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setMaxTries(0));
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setMaxTries(-1));
+    Crawler.builder().setMaxTries(1);
+  }
+
+  @Test
+  void testSetName() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setName(null));
+    Crawler.builder().setName("name");
+  }
+
+  @Test
+  void testSetParallelism() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setParallelism(0));
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setParallelism(-1));
+    Crawler.builder().setParallelism(1);
+  }
+
+  @Test
+  void testSetPropRetainProxy() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setPropRetainProxy(2));
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setPropRetainProxy(-1));
+    Crawler.builder().setPropRetainProxy(.05);
+  }
+
+  @Test
+  void testSetScheduler() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setScheduler(null));
+    Crawler.builder().setScheduler(new FIFOQueueScheduler());
+  }
+
+  @Test
+  void testSetSession() {
+    Crawler.builder().setSession(null);
+    Crawler.builder().setSession(Session.EMPTY_SESSION);
+  }
+
+  @Test
+  void testSetSleepScheduler() {
+    Crawler.builder().setSleepScheduler(null);
+    Crawler.builder().setSleepScheduler(new SleepScheduler(1));
+  }
+
+  @Test
+  void testSetWorkerManager() {
+    Assertions.assertThrows(IllegalStateException.class, () -> Crawler.builder().setWorkerManager(null));
+    Crawler.builder().setWorkerManager(new ThreadedWorkerManager());
+  }
+
+}
diff --git a/src/test/java/ai/preferred/venom/fetcher/AsyncFetcherBuilderTest.java b/src/test/java/ai/preferred/venom/fetcher/AsyncFetcherBuilderTest.java
new file mode 100644
index 0000000..875c3e3
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/fetcher/AsyncFetcherBuilderTest.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.fetcher;
+
+import ai.preferred.venom.UrlRouter;
+import ai.preferred.venom.request.Request;
+import ai.preferred.venom.response.Response;
+import ai.preferred.venom.storage.FileManager;
+import ai.preferred.venom.storage.Record;
+import ai.preferred.venom.validator.Validator;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nullable;
+import javax.net.ssl.SSLContext;
+import javax.validation.constraints.NotNull;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collections;
+
+public class AsyncFetcherBuilderTest {
+
+  @Test
+  void testDisableCompression() {
+    AsyncFetcher.builder().disableCompression();
+  }
+
+  @Test
+  void testDisableCookies() {
+    AsyncFetcher.builder().disableCookies();
+  }
+
+  @Test
+  void testRegister() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().register(null));
+    AsyncFetcher.builder().register(new Callback() {
+      @Override
+      public void completed(@NotNull Request request, @NotNull Response response) {
+
+      }
+
+      @Override
+      public void failed(@NotNull Request request, @NotNull Exception ex) {
+
+      }
+
+      @Override
+      public void cancelled(@NotNull Request request) {
+
+      }
+    });
+  }
+
+
+  @Test
+  void testSetRedirectStrategy() {
+    AsyncFetcher.builder().setRedirectStrategy(DefaultRedirectStrategy.INSTANCE);
+    AsyncFetcher.builder().setRedirectStrategy(null);
+  }
+
+  @Test
+  void testSetConnectionRequestTimeout() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setConnectionRequestTimeout(-2));
+    AsyncFetcher.builder().setConnectionRequestTimeout(-1);
+    AsyncFetcher.builder().setConnectionRequestTimeout(0);
+  }
+
+  @Test
+  void testSetConnectTimeout() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setConnectTimeout(-2));
+    AsyncFetcher.builder().setConnectTimeout(-1);
+    AsyncFetcher.builder().setConnectTimeout(0);
+  }
+
+  @Test
+  void testSetFileManager() {
+    AsyncFetcher.builder().setFileManager(null);
+    AsyncFetcher.builder().setFileManager(new FileManager() {
+      @Override
+      public @NotNull Callback getCallback() {
+        return null;
+      }
+
+      @Override
+      public @NotNull String put(@NotNull Request request, @NotNull Response response) {
+        return null;
+      }
+
+      @Nullable
+      @Override
+      public Record get(Object id) {
+        return null;
+      }
+
+      @Override
+      public @NotNull Record get(@NotNull Request request) {
+        return null;
+      }
+
+      @Override
+      public void close() {
+
+      }
+    });
+  }
+
+  @Test
+  void testSetHeaders() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setHeaders(null));
+    AsyncFetcher.builder().setHeaders(Collections.emptyMap());
+  }
+
+  @Test
+  void testSetMaxConnections() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setMaxConnections(-1));
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setMaxConnections(0));
+    AsyncFetcher.builder().setMaxConnections(1);
+  }
+
+  @Test
+  void testSetMaxRouteConnections() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setMaxRouteConnections(-1));
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setMaxRouteConnections(0));
+    AsyncFetcher.builder().setMaxRouteConnections(1);
+  }
+
+  @Test
+  void testSetNumIoThreads() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setNumIoThreads(0));
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setNumIoThreads(-1));
+    AsyncFetcher.builder().setNumIoThreads(1);
+  }
+
+  @Test
+  void testSetProxyProvider() {
+    AsyncFetcher.builder().setProxyProvider(null);
+    AsyncFetcher.builder().setProxyProvider(request -> null);
+  }
+
+  @Test
+  void testSetSocketTimeout() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setSocketTimeout(-2));
+    AsyncFetcher.builder().setSocketTimeout(-1);
+    AsyncFetcher.builder().setSocketTimeout(0);
+  }
+
+  @Test
+  void testSetSslContext() throws NoSuchAlgorithmException {
+    AsyncFetcher.builder().setSslContext(SSLContext.getDefault());
+    AsyncFetcher.builder().setSslContext(null);
+  }
+
+  @Test
+  void testSetStopCodes() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setStopCodes(null));
+    AsyncFetcher.builder().setStopCodes(500);
+  }
+
+  @Test
+  void testSetThreadFactory() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setThreadFactory(null));
+    AsyncFetcher.builder().setThreadFactory(r -> null);
+  }
+
+  @Test
+  void testSetValidator() {
+    AsyncFetcher.builder().setValidator(Validator.ALWAYS_VALID);
+  }
+
+  @Test
+  void testSetValidatorRouter() {
+    AsyncFetcher.builder().setValidatorRouter(null);
+    AsyncFetcher.builder().setValidatorRouter(new UrlRouter());
+  }
+
+  @Test
+  void testSetUserAgent() {
+    Assertions.assertThrows(IllegalStateException.class, () -> AsyncFetcher.builder().setUserAgent(null));
+    AsyncFetcher.builder().setUserAgent(() -> "");
+  }
+
+}

From 9600ebe2c5b546232079b5adfe18f73fcf12fbf9 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 12:37:27 +0800
Subject: [PATCH 10/17] Updates to job package

---
 src/main/java/ai/preferred/venom/Crawler.java |  10 +-
 .../job/AbstractPriorityQueueScheduler.java   |  61 +++++++++
 .../venom/job/AbstractQueueScheduler.java     |  56 ++++----
 .../java/ai/preferred/venom/job/BasicJob.java | 120 ------------------
 .../venom/job/FIFOQueueScheduler.java         |   5 +
 src/main/java/ai/preferred/venom/job/Job.java | 101 ++++++++++++---
 .../ai/preferred/venom/job/JobAttribute.java  |  23 ++++
 .../venom/job/LazyQueueScheduler.java         |  26 +---
 .../venom/job/PriorityJobAttribute.java       |  73 +++++++++++
 .../venom/job/PriorityQueueScheduler.java     |  21 +--
 .../preferred/venom/job/QueueScheduler.java   |  27 +++-
 .../ai/preferred/venom/job/Scheduler.java     |  66 +++-------
 .../venom/job/FIFOQueueSchedulerTest.java     |  11 +-
 .../venom/job/LazyQueueSchedulerTest.java     |  11 +-
 .../venom/job/PriorityQueueSchedulerTest.java |  46 +++++--
 src/test/resources/log4j.properties           |   5 +-
 16 files changed, 368 insertions(+), 294 deletions(-)
 create mode 100644 src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
 delete mode 100644 src/main/java/ai/preferred/venom/job/BasicJob.java
 create mode 100644 src/main/java/ai/preferred/venom/job/JobAttribute.java
 create mode 100644 src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java

diff --git a/src/main/java/ai/preferred/venom/Crawler.java b/src/main/java/ai/preferred/venom/Crawler.java
index b1d23e4..4ae5ce3 100644
--- a/src/main/java/ai/preferred/venom/Crawler.java
+++ b/src/main/java/ai/preferred/venom/Crawler.java
@@ -89,7 +89,7 @@ public final class Crawler implements Interruptible {
    * The scheduler used.
    */
   @NotNull
-  private final QueueScheduler<? extends Job> queueScheduler;
+  private final QueueScheduler queueScheduler;
 
   /**
    * The maximum number of simultaneous connections.
@@ -281,7 +281,9 @@ private void except(final Job job, final Throwable ex) {
       synchronized (pendingJobs) { // Synchronisation required to prevent crawler stopping incorrectly.
         pendingJobs.remove(job);
         if (job.getTryCount() < maxTries) {
-          job.reQueue();
+          job.prepareRetry();
+          queueScheduler.removeAndAdd(job);
+          LOGGER.debug("Job {} - {} re-queued.", Integer.toHexString(job.hashCode()), job.getRequest().getUrl());
         } else {
           LOGGER.error("Max retries reached for request: {}", job.getRequest().getUrl());
         }
@@ -558,7 +560,7 @@ public static final class Builder {
     /**
      * The scheduler used.
      */
-    private QueueScheduler<? extends Job> queueScheduler;
+    private QueueScheduler queueScheduler;
 
     /**
      * The sleep scheduler used.
@@ -649,7 +651,7 @@ public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
      * @param queueScheduler scheduler to be used.
      * @return this
      */
-    public Builder setScheduler(final @NotNull QueueScheduler<? extends Job> queueScheduler) {
+    public Builder setScheduler(final @NotNull QueueScheduler queueScheduler) {
       if (queueScheduler == null) {
         throw new IllegalStateException("Attribute 'queueScheduler' cannot be null.");
       }
diff --git a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
new file mode 100644
index 0000000..2a3f4bc
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import javax.annotation.Nonnull;
+import java.util.Comparator;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+public abstract class AbstractPriorityQueueScheduler extends AbstractQueueScheduler {
+
+  /**
+   * Constructs an instance of AbstractQueueScheduler.
+   */
+  protected AbstractPriorityQueueScheduler() {
+    super(new PriorityBlockingQueue<>(11,
+        Comparator.comparing(o -> ((PriorityJobAttribute) o.getJobAttribute(PriorityJobAttribute.class)))));
+  }
+
+  protected final Job ensurePriorityJobAttribute(final Job job) {
+    if (job.getJobAttribute(PriorityJobAttribute.class) == null) {
+      job.addJobAttribute(new PriorityJobAttribute());
+    }
+    return job;
+  }
+
+  @Override
+  public final void put(final @Nonnull Job job) throws InterruptedException {
+    ensurePriorityJobAttribute(job);
+    getQueue().put(job);
+  }
+
+  @Override
+  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
+      throws InterruptedException {
+    ensurePriorityJobAttribute(job);
+    return getQueue().offer(job, timeout, unit);
+  }
+
+  @Override
+  public final boolean offer(final @Nonnull Job job) {
+    ensurePriorityJobAttribute(job);
+    return getQueue().offer(job);
+  }
+
+
+}
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index f42f023..8f5ed3a 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -22,6 +22,7 @@
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
+import javax.validation.constraints.NotNull;
 import java.util.AbstractQueue;
 import java.util.Collection;
 import java.util.Iterator;
@@ -31,7 +32,7 @@
  * @author Ween Jiann Lee
  * @author Maksim Tkachenko
  */
-public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements QueueScheduler<Job> {
+public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements QueueScheduler {
 
   /**
    * The queue used for this scheduler.
@@ -50,7 +51,7 @@ public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implemen
    */
   protected AbstractQueueScheduler(final BlockingQueue<Job> queue) {
     this.queue = queue;
-    this.scheduler = new JobScheduler(queue);
+    this.scheduler = new JobScheduler(this);
   }
 
   @Override
@@ -90,11 +91,6 @@ public final int drainTo(final @Nonnull Collection<? super Job> c, final int max
     return queue.drainTo(c, maxElements);
   }
 
-  @Override
-  public final boolean offer(final @Nonnull Job job) {
-    return queue.offer(job);
-  }
-
   @Override
   public final Job peek() {
     return queue.peek();
@@ -110,7 +106,7 @@ protected final BlockingQueue<Job> getQueue() {
   }
 
   /**
-   * An implementation of ai.preferred.venom.job.Scheduler using BasicJob.
+   * An implementation of ai.preferred.venom.job.Scheduler using Job.
    */
   public static class JobScheduler implements Scheduler {
 
@@ -122,47 +118,43 @@ public static class JobScheduler implements Scheduler {
     /**
      * The queue used for this scheduler.
      */
-    private final BlockingQueue<Job> queue;
+    private final QueueScheduler queueScheduler;
 
     /**
      * Constructs an instance of JobScheduler.
      *
-     * @param queue an instance of BlockingQueue
+     * @param queueScheduler an instance of BlockingQueue
      */
-    public JobScheduler(final BlockingQueue<Job> queue) {
-      this.queue = queue;
-    }
-
-    @Override
-    public final void add(final Request r, final Handler h, final Priority p, final Priority pf) {
-      final Job job = new BasicJob(r, h, p, pf, queue);
-      queue.add(job);
-      LOGGER.debug("Added job {} - {} to queue.", Integer.toHexString(job.hashCode()), r.getUrl());
-    }
-
-    @Override
-    public final void add(final Request r, final Handler h, final Priority p) {
-      add(r, h, p, Priority.FLOOR);
+    public JobScheduler(final QueueScheduler queueScheduler) {
+      this.queueScheduler = queueScheduler;
     }
 
     @Override
-    public final void add(final Request r, final Handler h) {
-      add(r, h, Priority.DEFAULT);
+    public void add(final @NotNull Request request, final @NotNull Handler handler,
+                    final JobAttribute... jobAttributes) {
+      final Job job = new Job(request, handler);
+      if (jobAttributes != null) {
+        for (JobAttribute jobAttribute : jobAttributes) {
+          job.addJobAttribute(jobAttribute);
+        }
+      }
+      queueScheduler.add(job);
+      LOGGER.debug("Job {} - {} added to queue.", job.toString(), request.getUrl());
     }
 
     @Override
-    public final void add(final Request r, final Priority p, final Priority pf) {
-      add(r, null, p, pf);
+    public void add(final @NotNull Request request, final JobAttribute... jobAttributes) {
+      add(request, null, jobAttributes);
     }
 
     @Override
-    public final void add(final Request r, final Priority p) {
-      add(r, null, p, Priority.FLOOR);
+    public final void add(final Request request, final Handler handler) {
+      add(request, handler, (JobAttribute[]) null);
     }
 
     @Override
-    public final void add(final Request r) {
-      add(r, null, Priority.DEFAULT, Priority.FLOOR);
+    public void add(final @NotNull Request request) {
+      add(request, null, (JobAttribute[]) null);
     }
 
   }
diff --git a/src/main/java/ai/preferred/venom/job/BasicJob.java b/src/main/java/ai/preferred/venom/job/BasicJob.java
deleted file mode 100644
index 328a759..0000000
--- a/src/main/java/ai/preferred/venom/job/BasicJob.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright 2018 Preferred.AI
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ai.preferred.venom.job;
-
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nonnull;
-import java.util.Queue;
-
-/**
- * @author Ween Jiann Lee
- */
-public class BasicJob implements Job {
-
-  /**
-   * Logger.
-   */
-  private static final Logger LOGGER = LoggerFactory.getLogger(BasicJob.class);
-
-  /**
-   * The request of this job.
-   */
-  private final Request request;
-
-  /**
-   * The handler of this job.
-   */
-  private final Handler handler;
-
-  /**
-   * The priority floor of this job.
-   */
-  private final Priority priorityFloor;
-
-  /**
-   * The queue for this job.
-   */
-  private final Queue<Job> queue;
-
-  /**
-   * The priority  of this job.
-   */
-  private Priority priority;
-
-  /**
-   * The current try of this job.
-   */
-  private int tryCount = 1;
-
-  /**
-   * Constructs a basic job.
-   *
-   * @param request       The request of this job.
-   * @param handler       The handler of this job.
-   * @param priority      The priority of this job.
-   * @param priorityFloor The priority floor of this job.
-   * @param queue         The queue for this job.
-   */
-  public BasicJob(final Request request, final Handler handler, final Priority priority,
-                  final Priority priorityFloor, final Queue<Job> queue) {
-    this.request = request;
-    this.handler = handler;
-    this.priority = priority;
-    this.priorityFloor = priorityFloor;
-    this.queue = queue;
-  }
-
-  @Override
-  public final Request getRequest() {
-    return request;
-  }
-
-  @Override
-  public final Handler getHandler() {
-    return handler;
-  }
-
-  @Override
-  public final Priority getPriority() {
-    return priority;
-  }
-
-  @Override
-  public final void reQueue() {
-    queue.remove(this);
-    priority = priority.downgrade(priorityFloor);
-    tryCount++;
-    queue.add(this);
-
-    LOGGER.debug("Re-queued job {} - {}.", Integer.toHexString(hashCode()), request.getUrl());
-  }
-
-  @Override
-  public final int getTryCount() {
-    return tryCount;
-  }
-
-  @Override
-  public final int compareTo(final @Nonnull Job job) {
-    return priority.compareTo(job.getPriority());
-  }
-
-}
diff --git a/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
index 6876bc4..db9d411 100644
--- a/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/FIFOQueueScheduler.java
@@ -49,6 +49,11 @@ public final boolean offer(final Job job, final long timeout, final @Nonnull Tim
     return getQueue().offer(job, timeout, unit);
   }
 
+  @Override
+  public final boolean offer(final @Nonnull Job job) {
+    return getQueue().offer(job);
+  }
+
   @Override
   public final Job poll(final long timeout, final @Nonnull TimeUnit unit) throws InterruptedException {
     return getQueue().poll(timeout, unit);
diff --git a/src/main/java/ai/preferred/venom/job/Job.java b/src/main/java/ai/preferred/venom/job/Job.java
index e1ac770..e9a9e87 100644
--- a/src/main/java/ai/preferred/venom/job/Job.java
+++ b/src/main/java/ai/preferred/venom/job/Job.java
@@ -18,18 +18,70 @@
 
 import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.Request;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
- * This interface represents only the most basic of a job, to
- * be placed in a scheduler or other forms.
- *
- * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public interface Job extends Comparable<Job> {
+public class Job {
+
+  /**
+   * Logger.
+   */
+  private static final Logger LOGGER = LoggerFactory.getLogger(Job.class);
+
+  /**
+   * The request of this job.
+   */
+  private final Request request;
+
+  /**
+   * The handler of this job.
+   */
+  private final Handler handler;
+
+  /**
+   *
+   */
+  private final Map<Class<? extends JobAttribute>, JobAttribute> jobAttributeMap = new HashMap<>();
+
+  /**
+   * The current try of this job.
+   */
+  private int tryCount = 1;
+
+  /**
+   * Constructs a basic job.
+   *
+   * @param request The request of this job.
+   * @param handler The handler of this job.
+   */
+  public Job(final Request request, final Handler handler) {
+    this(request, handler, (JobAttribute[]) null);
+  }
+
+  /**
+   * Constructs a basic job.
+   *
+   * @param request The request of this job.
+   * @param handler The handler of this job.
+   */
+  public Job(final Request request, final Handler handler, final JobAttribute... jobAttributes) {
+    this.request = request;
+    this.handler = handler;
+    if (jobAttributes != null) {
+      for (final JobAttribute jobAttribute : jobAttributes) {
+        jobAttributeMap.put(jobAttribute.getClass(), jobAttribute);
+      }
+    }
+  }
+
 
   /**
    * Get the request of this job.
@@ -37,7 +89,9 @@ public interface Job extends Comparable<Job> {
    * @return Request of the job.
    */
   @NotNull
-  Request getRequest();
+  public final Request getRequest() {
+    return request;
+  }
 
   /**
    * Get the handler to handle the response of the job.
@@ -49,26 +103,31 @@ public interface Job extends Comparable<Job> {
    * @return Handler for the response or null.
    */
   @Nullable
-  Handler getHandler();
-
-  /**
-   * Get the current priority set for this job.
-   *
-   * @return the current priority of the job.
-   */
-  Priority getPriority();
-
-  /**
-   * Remove any existing in queue, downgrades the priority and
-   * adds the job back into queue.
-   */
-  void reQueue();
+  public final Handler getHandler() {
+    return handler;
+  }
 
   /**
    * Get attempt number of this job.
    *
    * @return Attempt (try) count of the job.
    */
-  int getTryCount();
+  public final int getTryCount() {
+    return tryCount;
+  }
+
+  public final void prepareRetry() {
+    jobAttributeMap.forEach((k, jobAttribute) -> jobAttribute.prepareRetry());
+    tryCount++;
+  }
+
+  public final Job addJobAttribute(final JobAttribute jobAttribute) {
+    jobAttributeMap.put(jobAttribute.getClass(), jobAttribute);
+    return this;
+  }
+
+  public final JobAttribute getJobAttribute(final Class<? extends JobAttribute> clazz) {
+    return jobAttributeMap.get(clazz);
+  }
 
 }
diff --git a/src/main/java/ai/preferred/venom/job/JobAttribute.java b/src/main/java/ai/preferred/venom/job/JobAttribute.java
new file mode 100644
index 0000000..ef31288
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/JobAttribute.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+public interface JobAttribute {
+
+  void prepareRetry();
+
+}
diff --git a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
index a697c50..dd7a7bd 100644
--- a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
@@ -22,7 +22,6 @@
 
 import javax.annotation.Nonnull;
 import java.util.Iterator;
-import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -36,7 +35,7 @@
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public class LazyQueueScheduler extends AbstractQueueScheduler {
+public class LazyQueueScheduler extends AbstractPriorityQueueScheduler {
 
   /**
    * An object to synchronise upon.
@@ -60,7 +59,7 @@ public class LazyQueueScheduler extends AbstractQueueScheduler {
    * @param handler  The default handler to use
    */
   public LazyQueueScheduler(final Iterator<Request> requests, final Handler handler) {
-    super(new PriorityBlockingQueue<>());
+
     this.requests = requests;
     this.handler = handler;
   }
@@ -80,38 +79,27 @@ public LazyQueueScheduler(final Iterator<Request> requests) {
    * @return An new job instance
    */
   private Job pollLazyRequest() {
-    return new BasicJob(requests.next(), handler, Priority.DEFAULT, Priority.FLOOR, getQueue());
+    return new Job(requests.next(), handler, new PriorityJobAttribute());
   }
 
   @Override
-  public final Job poll() {
+  public final Job poll(final long time, final @Nonnull TimeUnit unit) throws InterruptedException {
     synchronized (lock) {
       if (getQueue().isEmpty() && requests.hasNext()) {
         return pollLazyRequest();
       }
     }
-    return getQueue().poll();
-  }
-
-  @Override
-  public final void put(final @Nonnull Job job) throws InterruptedException {
-    getQueue().put(job);
-  }
-
-  @Override
-  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
-      throws InterruptedException {
-    return getQueue().offer(job, timeout, unit);
+    return getQueue().poll(time, unit);
   }
 
   @Override
-  public final Job poll(final long time, final @Nonnull TimeUnit unit) throws InterruptedException {
+  public final Job poll() {
     synchronized (lock) {
       if (getQueue().isEmpty() && requests.hasNext()) {
         return pollLazyRequest();
       }
     }
-    return getQueue().poll(time, unit);
+    return getQueue().poll();
   }
 
   @Override
diff --git a/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java b/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java
new file mode 100644
index 0000000..e8e3d75
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import javax.annotation.Nonnull;
+
+public class PriorityJobAttribute implements JobAttribute, Comparable<PriorityJobAttribute> {
+
+  /**
+   * The priority floor of this job.
+   */
+  private final Priority priorityFloor;
+
+  /**
+   * The priority  of this job.
+   */
+  private Priority priority;
+
+  /**
+   * Constructs an instance of PriorityJobAttribute.
+   *
+   * @param priority      The priority of this job.
+   * @param priorityFloor The priority floor of this job.
+   */
+  public PriorityJobAttribute(final Priority priority, final Priority priorityFloor) {
+    this.priority = priority;
+    this.priorityFloor = priorityFloor;
+  }
+
+  /**
+   * Constructs an instance of PriorityJobAttribute.
+   *
+   * @param priority The priority of this job.
+   */
+  public PriorityJobAttribute(final Priority priority) {
+    this(priority, Priority.FLOOR);
+  }
+
+  /**
+   * Constructs an instance of PriorityJobAttribute.
+   */
+  public PriorityJobAttribute() {
+    this(Priority.DEFAULT);
+  }
+
+  public final Priority getPriority() {
+    return priority;
+  }
+
+  @Override
+  public void prepareRetry() {
+    priority = priority.downgrade(priorityFloor);
+  }
+
+  @Override
+  public final int compareTo(final @Nonnull PriorityJobAttribute job) {
+    return priority.compareTo(job.getPriority());
+  }
+}
diff --git a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
index 834eee0..7fb969c 100644
--- a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
@@ -18,7 +18,6 @@
 
 
 import javax.annotation.Nonnull;
-import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -31,25 +30,7 @@
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public class PriorityQueueScheduler extends AbstractQueueScheduler {
-
-  /**
-   * The queue used for this scheduler.
-   */
-  public PriorityQueueScheduler() {
-    super(new PriorityBlockingQueue<>());
-  }
-
-  @Override
-  public final void put(final @Nonnull Job job) throws InterruptedException {
-    getQueue().put(job);
-  }
-
-  @Override
-  public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
-      throws InterruptedException {
-    return getQueue().offer(job, timeout, unit);
-  }
+public class PriorityQueueScheduler extends AbstractPriorityQueueScheduler {
 
   @Override
   public final Job poll(final long timeout, final @Nonnull TimeUnit unit) throws InterruptedException {
diff --git a/src/main/java/ai/preferred/venom/job/QueueScheduler.java b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
index ac7cd25..da620e0 100644
--- a/src/main/java/ai/preferred/venom/job/QueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
@@ -17,6 +17,7 @@
 package ai.preferred.venom.job;
 
 
+import javax.annotation.Nonnull;
 import java.util.concurrent.BlockingQueue;
 
 /**
@@ -24,11 +25,10 @@
  * It imposes no restrictions or particular details on the the
  * type of queue, and allows for different future types to be returned.
  *
- * @param <T> the type of Job to schedule
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public interface QueueScheduler<T extends Job> extends BlockingQueue<T> {
+public interface QueueScheduler extends BlockingQueue<Job> {
 
   /**
    * Get the scheduler to add jobs.
@@ -37,4 +37,27 @@ public interface QueueScheduler<T extends Job> extends BlockingQueue<T> {
    */
   Scheduler getScheduler();
 
+  /**
+   * Removes a single instance of the specified element from this queue,
+   * if it is present then Inserts the specified element into this queue
+   * if it is possible to do so immediately without violating capacity
+   * restrictions, returning {@code true} upon success and throwing an
+   * {@code IllegalStateException} if no space is currently available.
+   *
+   * @param job element to be removed from this queue, if present and added
+   * @throws ClassCastException       if the class of the specified element
+   *                                  is incompatible with this queue or if the class
+   *                                  of the specified element
+   *                                  prevents it from being added to this queue
+   * @throws NullPointerException     if the specified element is null
+   * @throws IllegalArgumentException if some property of the specified
+   *                                  element prevents it from being added to this queue
+   */
+  default void removeAndAdd(final @Nonnull Job job) {
+    synchronized (job) {
+      remove(job);
+      add(job);
+    }
+  }
+
 }
diff --git a/src/main/java/ai/preferred/venom/job/Scheduler.java b/src/main/java/ai/preferred/venom/job/Scheduler.java
index 2c812d8..7e38fce 100644
--- a/src/main/java/ai/preferred/venom/job/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/Scheduler.java
@@ -32,80 +32,46 @@ public interface Scheduler {
   /**
    * Adds a request to the queue.
    * <p>
-   * This request would be parsed by the handler specified, and
-   * its priority can be downgraded to a minimum priority specified.
+   * This request would be parsed by the handler specified.
    * </p>
    *
-   * @param r  request to fetch when dequeued
-   * @param h  handler to be used to parse the request
-   * @param p  initial priority of the request
-   * @param pf the minimum (floor) priority of this request
+   * @param request       request to fetch when dequeued.
+   * @param handler       handler to be used to parse the request.
+   * @param jobAttributes attributes to insert to the job.
    */
-  void add(@NotNull Request r, @NotNull Handler h, Priority p, Priority pf);
+  void add(@NotNull Request request, @NotNull Handler handler, JobAttribute... jobAttributes);
 
   /**
    * Adds a request to the queue.
    * <p>
-   * This request would be parsed by the handler specified, and
-   * its priority can be downgraded to the default minimum priority.
+   * This request would be parsed by the handler specified.
    * </p>
    *
-   * @param r request to fetch when dequeued
-   * @param h handler to be used to parse the request
-   * @param p initial priority of the request
+   * @param request       request to fetch when dequeued.
+   * @param jobAttributes attributes to insert to the job.
    */
-  void add(@NotNull Request r, @NotNull Handler h, Priority p);
+  void add(@NotNull Request request, JobAttribute... jobAttributes);
 
   /**
    * Adds a request to the queue.
    * <p>
-   * This request would be parsed by the handler specified, and
-   * it's initialised with default priority that can be downgraded to
-   * the default minimum priority.
+   * This request would be parsed by the handler specified.
    * </p>
    *
-   * @param r request to fetch when dequeued
-   * @param h handler to be used to parse the request
+   * @param request request to fetch when dequeued.
+   * @param handler handler to be used to parse the request.
    */
-  void add(@NotNull Request r, @NotNull Handler h);
+  void add(@NotNull Request request, @NotNull Handler handler);
 
   /**
    * Adds a request to the queue.
    * <p>
    * This request would be parsed by a handler defined in Router
-   * or otherwise, and its priority can be downgraded to a minimum
-   * priority specified.
+   * or otherwise defined.
    * </p>
    *
-   * @param r  request to fetch when dequeued
-   * @param p  initial priority of the request
-   * @param pf the minimum (floor) priority of this request
+   * @param request request to fetch when dequeued.
    */
-  void add(@NotNull Request r, Priority p, Priority pf);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by a handler defined in Router
-   * or otherwise defined, and its priority can be downgraded to the
-   * default minimum priority.
-   * </p>
-   *
-   * @param r request to fetch when dequeued
-   * @param p initial priority of the request
-   */
-  void add(@NotNull Request r, Priority p);
-
-  /**
-   * Adds a request to the queue.
-   * <p>
-   * This request would be parsed by a handler defined in Router
-   * or otherwise defined, and it's initialised with default priority
-   * that can be downgraded to the default minimum priority.
-   * </p>
-   *
-   * @param r request to fetch when dequeued
-   */
-  void add(@NotNull Request r);
+  void add(@NotNull Request request);
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
index e4cfdfa..36f7da4 100644
--- a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
@@ -35,7 +35,6 @@ public void testAddRequest() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
   }
 
   @Test
@@ -54,7 +53,6 @@ public void testAddRequestHandler() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertEquals(handler, job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
   }
 
   @Test
@@ -65,16 +63,15 @@ public void testFIFOQueue() {
     final VRequest vRequest = new VRequest(url);
     final VRequest vRequestNeg = new VRequest(url);
 
-    scheduler.getScheduler().add(vRequest, Priority.HIGH);
-    scheduler.getScheduler().add(vRequestNeg, Priority.HIGHEST);
-    scheduler.getScheduler().add(vRequestNeg, Priority.DEFAULT);
-    scheduler.getScheduler().add(vRequestNeg, Priority.LOW);
+    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGH));
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.HIGHEST));
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.DEFAULT));
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.LOW));
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.HIGH, job.getPriority());
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
index 16fc93f..66dcc3b 100644
--- a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
@@ -50,7 +50,10 @@ public void testIterator() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertEquals(handler, job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
+    Assertions.assertEquals(
+        Priority.DEFAULT,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
   @Test
@@ -69,7 +72,6 @@ public void testLazyQueue() {
 
     };
 
-
     final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
     final VRequest vRequest = new VRequest(url);
     scheduler.getScheduler().add(vRequest);
@@ -78,7 +80,10 @@ public void testLazyQueue() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
+    Assertions.assertEquals(
+        Priority.DEFAULT,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
index f4e034c..80cf0a1 100644
--- a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
@@ -35,7 +35,10 @@ public void testAddRequest() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
+    Assertions.assertEquals(
+        Priority.DEFAULT,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
   @Test
@@ -54,7 +57,10 @@ public void testAddRequestHandler() {
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertEquals(handler, job.getHandler());
-    Assertions.assertEquals(Priority.DEFAULT, job.getPriority());
+    Assertions.assertEquals(
+        Priority.DEFAULT,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
   @Test
@@ -65,16 +71,19 @@ public void testPriority() {
     final VRequest vRequest = new VRequest(url);
     final VRequest vRequestNeg = new VRequest(url);
 
-    scheduler.getScheduler().add(vRequestNeg, Priority.HIGH);
-    scheduler.getScheduler().add(vRequest, Priority.HIGHEST);
-    scheduler.getScheduler().add(vRequestNeg, Priority.DEFAULT);
-    scheduler.getScheduler().add(vRequestNeg, Priority.LOW);
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.HIGH));
+    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGHEST));
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.DEFAULT));
+    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.LOW));
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.HIGHEST, job.getPriority());
+    Assertions.assertEquals(
+        Priority.HIGHEST,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
   @Test
@@ -84,27 +93,38 @@ public void testPriorityFloor() {
     final String url = "https://venom.preferred.ai";
     final VRequest vRequest = new VRequest(url);
 
-    scheduler.getScheduler().add(vRequest, Priority.HIGH, Priority.NORMAL);
+    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGH, Priority.NORMAL));
 
     final Job job = scheduler.poll();
     Assertions.assertNotNull(job);
     Assertions.assertEquals(vRequest, job.getRequest());
     Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(Priority.HIGH, job.getPriority());
+    Assertions.assertEquals(
+        Priority.HIGH,
+        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
 
-    job.reQueue();
+    job.prepareRetry();
+    scheduler.add(job);
     final Job jobRQ = scheduler.poll();
     Assertions.assertNotNull(jobRQ);
     Assertions.assertEquals(vRequest, jobRQ.getRequest());
     Assertions.assertNull(jobRQ.getHandler());
-    Assertions.assertEquals(Priority.NORMAL, jobRQ.getPriority());
+    Assertions.assertEquals(
+        Priority.NORMAL,
+        ((PriorityJobAttribute) jobRQ.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
 
-    job.reQueue();
+    job.prepareRetry();
+    scheduler.add(job);
     final Job jobRQRQ = scheduler.poll();
     Assertions.assertNotNull(jobRQRQ);
     Assertions.assertEquals(vRequest, jobRQRQ.getRequest());
     Assertions.assertNull(jobRQRQ.getHandler());
-    Assertions.assertEquals(Priority.NORMAL, jobRQRQ.getPriority());
+    Assertions.assertEquals(
+        Priority.NORMAL,
+        ((PriorityJobAttribute) jobRQRQ.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+    );
   }
 
 }
diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties
index 55e30d4..a26f350 100644
--- a/src/test/resources/log4j.properties
+++ b/src/test/resources/log4j.properties
@@ -13,12 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 log4j.rootLogger=ERROR, STDOUT
+# Venom
 log4j.logger.ai.preferred.venom=DEBUG, STDOUT
-
 log4j.additivity.ai.preferred.venom=false
-
+# Appender
 log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
 log4j.appender.STDOUT.Target=System.out
 log4j.appender.STDOUT.layout=org.apache.log4j.EnhancedPatternLayout

From 8f45e1c8d0fb1953c862454a57a2a14b425d2e54 Mon Sep 17 00:00:00 2001
From: Lee Ween Jiann <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 12:56:01 +0800
Subject: [PATCH 11/17] Updated Javadoc

---
 .../job/AbstractPriorityQueueScheduler.java   | 21 ++++++++++-----
 .../venom/job/AbstractQueueScheduler.java     |  8 +++---
 src/main/java/ai/preferred/venom/job/Job.java | 27 +++++++++++++++++--
 .../ai/preferred/venom/job/JobAttribute.java  | 14 ++++++++++
 .../venom/job/PriorityJobAttribute.java       | 13 ++++++++-
 .../venom/job/PriorityQueueScheduler.java     |  2 +-
 6 files changed, 70 insertions(+), 15 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
index 2a3f4bc..1a4f383 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
@@ -21,6 +21,9 @@
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+/**
+ * @author Ween Jiann Lee
+ */
 public abstract class AbstractPriorityQueueScheduler extends AbstractQueueScheduler {
 
   /**
@@ -31,7 +34,14 @@ protected AbstractPriorityQueueScheduler() {
         Comparator.comparing(o -> ((PriorityJobAttribute) o.getJobAttribute(PriorityJobAttribute.class)))));
   }
 
-  protected final Job ensurePriorityJobAttribute(final Job job) {
+  /**
+   * Check the job for {@see PriorityJobAttribute}, if missing,
+   * adds it to the job.
+   *
+   * @param job the job to check.
+   * @return the input job.
+   */
+  private Job ensurePriorityJobAttribute(final Job job) {
     if (job.getJobAttribute(PriorityJobAttribute.class) == null) {
       job.addJobAttribute(new PriorityJobAttribute());
     }
@@ -40,21 +50,18 @@ protected final Job ensurePriorityJobAttribute(final Job job) {
 
   @Override
   public final void put(final @Nonnull Job job) throws InterruptedException {
-    ensurePriorityJobAttribute(job);
-    getQueue().put(job);
+    getQueue().put(ensurePriorityJobAttribute(job));
   }
 
   @Override
   public final boolean offer(final Job job, final long timeout, final @Nonnull TimeUnit unit)
       throws InterruptedException {
-    ensurePriorityJobAttribute(job);
-    return getQueue().offer(job, timeout, unit);
+    return getQueue().offer(ensurePriorityJobAttribute(job), timeout, unit);
   }
 
   @Override
   public final boolean offer(final @Nonnull Job job) {
-    ensurePriorityJobAttribute(job);
-    return getQueue().offer(job);
+    return getQueue().offer(ensurePriorityJobAttribute(job));
   }
 
 
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 8f5ed3a..159ac93 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -130,8 +130,8 @@ public JobScheduler(final QueueScheduler queueScheduler) {
     }
 
     @Override
-    public void add(final @NotNull Request request, final @NotNull Handler handler,
-                    final JobAttribute... jobAttributes) {
+    public final void add(final @NotNull Request request, final @NotNull Handler handler,
+                          final JobAttribute... jobAttributes) {
       final Job job = new Job(request, handler);
       if (jobAttributes != null) {
         for (JobAttribute jobAttribute : jobAttributes) {
@@ -143,7 +143,7 @@ public void add(final @NotNull Request request, final @NotNull Handler handler,
     }
 
     @Override
-    public void add(final @NotNull Request request, final JobAttribute... jobAttributes) {
+    public final void add(final @NotNull Request request, final JobAttribute... jobAttributes) {
       add(request, null, jobAttributes);
     }
 
@@ -153,7 +153,7 @@ public final void add(final Request request, final Handler handler) {
     }
 
     @Override
-    public void add(final @NotNull Request request) {
+    public final void add(final @NotNull Request request) {
       add(request, null, (JobAttribute[]) null);
     }
 
diff --git a/src/main/java/ai/preferred/venom/job/Job.java b/src/main/java/ai/preferred/venom/job/Job.java
index e9a9e87..83de745 100644
--- a/src/main/java/ai/preferred/venom/job/Job.java
+++ b/src/main/java/ai/preferred/venom/job/Job.java
@@ -69,8 +69,9 @@ public Job(final Request request, final Handler handler) {
   /**
    * Constructs a basic job.
    *
-   * @param request The request of this job.
-   * @param handler The handler of this job.
+   * @param request       The request of this job.
+   * @param handler       The handler of this job.
+   * @param jobAttributes attributes to insert to the job.
    */
   public Job(final Request request, final Handler handler, final JobAttribute... jobAttributes) {
     this.request = request;
@@ -116,16 +117,38 @@ public final int getTryCount() {
     return tryCount;
   }
 
+  /**
+   * This method is called before the job is scheduled
+   * for a retry.
+   * <p>
+   * This method allows you to specify the logic to
+   * move the job into its subsequent state for a retry.
+   * </p>
+   */
   public final void prepareRetry() {
     jobAttributeMap.forEach((k, jobAttribute) -> jobAttribute.prepareRetry());
     tryCount++;
   }
 
+  /**
+   * Adds or replace the current job attribute if the class of
+   * attribute is already present in the map.
+   *
+   * @param jobAttribute the job attribute to add or replace.
+   * @return this.
+   */
   public final Job addJobAttribute(final JobAttribute jobAttribute) {
     jobAttributeMap.put(jobAttribute.getClass(), jobAttribute);
     return this;
   }
 
+  /**
+   * Get the job attribute for a specific attribute class or
+   * return {@code null} if not found.
+   *
+   * @param clazz The class of attribute to find.
+   * @return an instance of job attribute for class or null.
+   */
   public final JobAttribute getJobAttribute(final Class<? extends JobAttribute> clazz) {
     return jobAttributeMap.get(clazz);
   }
diff --git a/src/main/java/ai/preferred/venom/job/JobAttribute.java b/src/main/java/ai/preferred/venom/job/JobAttribute.java
index ef31288..55ee961 100644
--- a/src/main/java/ai/preferred/venom/job/JobAttribute.java
+++ b/src/main/java/ai/preferred/venom/job/JobAttribute.java
@@ -16,8 +16,22 @@
 
 package ai.preferred.venom.job;
 
+/**
+ * This interface represents attributes that can be added
+ * to jobs to manipulate the crawling process.
+ *
+ * @author Ween Jiann Lee
+ */
 public interface JobAttribute {
 
+  /**
+   * This method is called before the job is scheduled
+   * for a retry.
+   * <p>
+   * This method allows you to specify the logic to
+   * move the job into its subsequent state for a retry.
+   * </p>
+   */
   void prepareRetry();
 
 }
diff --git a/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java b/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java
index e8e3d75..a4852f1 100644
--- a/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java
+++ b/src/main/java/ai/preferred/venom/job/PriorityJobAttribute.java
@@ -18,6 +18,12 @@
 
 import javax.annotation.Nonnull;
 
+/**
+ * This class provides an implementation of job attribute with comparable
+ * priority.
+ *
+ * @author Ween Jiann Lee
+ */
 public class PriorityJobAttribute implements JobAttribute, Comparable<PriorityJobAttribute> {
 
   /**
@@ -57,12 +63,17 @@ public PriorityJobAttribute() {
     this(Priority.DEFAULT);
   }
 
+  /**
+   * Get the priority in this attribute.
+   *
+   * @return the priority in this attribute.
+   */
   public final Priority getPriority() {
     return priority;
   }
 
   @Override
-  public void prepareRetry() {
+  public final void prepareRetry() {
     priority = priority.downgrade(priorityFloor);
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
index 7fb969c..5cfd048 100644
--- a/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/PriorityQueueScheduler.java
@@ -21,7 +21,7 @@
 import java.util.concurrent.TimeUnit;
 
 /**
- * This class provides and implementation of scheduler with a priority
+ * This class provides an implementation of scheduler with a priority
  * sensitive queue.
  * <p>
  * Jobs with higher priority will be processed first.

From 66ff0b19b044cb453d21435f2c93decadc0510d1 Mon Sep 17 00:00:00 2001
From: Lee Ween Jiann <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 13:12:06 +0800
Subject: [PATCH 12/17] Fixed warnings

---
 .../venom/job/AbstractQueueScheduler.java     |  1 +
 .../preferred/venom/job/QueueScheduler.java   |  1 +
 .../venom/storage/DummyFileManagerTest.java   |  4 +---
 .../venom/storage/MysqlFileManagerTest.java   |  7 ++-----
 .../validator/EmptyContentValidatorTest.java  |  6 ++----
 .../validator/MimeTypeValidatorTest.java      | 10 ++++------
 .../validator/PipelineValidatorTest.java      | 20 +++++++++----------
 .../validator/StatusOkValidatorTest.java      |  6 ++----
 8 files changed, 22 insertions(+), 33 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 159ac93..9323b53 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -32,6 +32,7 @@
  * @author Ween Jiann Lee
  * @author Maksim Tkachenko
  */
+@SuppressWarnings("NullableProblems")
 public abstract class AbstractQueueScheduler extends AbstractQueue<Job> implements QueueScheduler {
 
   /**
diff --git a/src/main/java/ai/preferred/venom/job/QueueScheduler.java b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
index da620e0..b54d6f3 100644
--- a/src/main/java/ai/preferred/venom/job/QueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/QueueScheduler.java
@@ -53,6 +53,7 @@ public interface QueueScheduler extends BlockingQueue<Job> {
    * @throws IllegalArgumentException if some property of the specified
    *                                  element prevents it from being added to this queue
    */
+  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
   default void removeAndAdd(final @Nonnull Job job) {
     synchronized (job) {
       remove(job);
diff --git a/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java b/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java
index 12e83a7..beeb237 100644
--- a/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java
+++ b/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java
@@ -22,7 +22,6 @@
 import ai.preferred.venom.response.Response;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.*;
 
@@ -81,10 +80,9 @@ void testPut() throws StorageException {
     final byte[] content = "This is test data.".getBytes();
     final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
     final Header[] headers = {};
-    final HttpHost proxy = null;
 
     final Request request = new VRequest(url);
-    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, null);
 
     final String md5 = DigestUtils.md5Hex(content);
     final String subDirName = md5.substring(0, 2);
diff --git a/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java b/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java
index 23da045..962f11e 100644
--- a/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java
+++ b/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java
@@ -23,7 +23,6 @@
 import ch.vorburger.exec.ManagedProcessException;
 import ch.vorburger.mariadb4j.DB;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -98,10 +97,9 @@ void testPut() throws StorageException {
     final byte[] content = "This is put test data.".getBytes();
     final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
     final Header[] headers = {};
-    final HttpHost proxy = null;
 
     final Request request = new VRequest(url);
-    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, null);
 
     fileManager.put(request, response);
     // TODO: Check DB
@@ -114,10 +112,9 @@ void testCompleteCallback() {
     final byte[] content = "This is complete callback test data.".getBytes();
     final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
     final Header[] headers = {};
-    final HttpHost proxy = null;
 
     final Request request = new VRequest(url);
-    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, url, content, contentType, headers, null);
 
     fileManager.getCallback().completed(request, response);
     // TODO: Check DB
diff --git a/src/test/java/ai/preferred/venom/validator/EmptyContentValidatorTest.java b/src/test/java/ai/preferred/venom/validator/EmptyContentValidatorTest.java
index acb449e..d1ade7f 100644
--- a/src/test/java/ai/preferred/venom/validator/EmptyContentValidatorTest.java
+++ b/src/test/java/ai/preferred/venom/validator/EmptyContentValidatorTest.java
@@ -21,7 +21,6 @@
 import ai.preferred.venom.response.BaseResponse;
 import ai.preferred.venom.response.Response;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -35,19 +34,18 @@ public class EmptyContentValidatorTest {
   private final String baseUrl = "https://venom.preferred.ai";
   private final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
   private final Header[] headers = {};
-  private final HttpHost proxy = null;
 
   @Test
   public void testEmptyContent() {
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.INVALID_CONTENT, EmptyContentValidator.INSTANCE.isValid(request, response));
   }
 
   @Test
   public void testValidContent() {
     final byte[] content = "IPSUM".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.VALID, EmptyContentValidator.INSTANCE.isValid(request, response));
   }
 
diff --git a/src/test/java/ai/preferred/venom/validator/MimeTypeValidatorTest.java b/src/test/java/ai/preferred/venom/validator/MimeTypeValidatorTest.java
index 0c3373f..a2a5af0 100644
--- a/src/test/java/ai/preferred/venom/validator/MimeTypeValidatorTest.java
+++ b/src/test/java/ai/preferred/venom/validator/MimeTypeValidatorTest.java
@@ -21,7 +21,6 @@
 import ai.preferred.venom.response.BaseResponse;
 import ai.preferred.venom.response.Response;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -36,12 +35,11 @@ public class MimeTypeValidatorTest {
   private final String baseUrl = "https://venom.preferred.ai";
   private final byte[] content = "IPSUM".getBytes();
   private final Header[] headers = {};
-  private final HttpHost proxy = null;
 
   @Test
   public void testValidMimeTypePattern() {
     final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.VALID,
         new MimeTypeValidator(Pattern.compile("^text.*")).isValid(request, response));
   }
@@ -49,7 +47,7 @@ public void testValidMimeTypePattern() {
   @Test
   public void testInvalidMimeTypePattern() {
     final ContentType contentType = ContentType.create("text/json", StandardCharsets.UTF_8);
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.INVALID_CONTENT,
         new MimeTypeValidator(Pattern.compile("^image.*")).isValid(request, response));
   }
@@ -57,7 +55,7 @@ public void testInvalidMimeTypePattern() {
   @Test
   public void testValidMimeTypeString() {
     final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.VALID,
         new MimeTypeValidator("^text.*").isValid(request, response));
   }
@@ -65,7 +63,7 @@ public void testValidMimeTypeString() {
   @Test
   public void testInvalidMimeTypeString() {
     final ContentType contentType = ContentType.create("text/json", StandardCharsets.UTF_8);
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.INVALID_CONTENT,
         new MimeTypeValidator("^image.*").isValid(request, response));
   }
diff --git a/src/test/java/ai/preferred/venom/validator/PipelineValidatorTest.java b/src/test/java/ai/preferred/venom/validator/PipelineValidatorTest.java
index 7f2606d..3ceeb1c 100644
--- a/src/test/java/ai/preferred/venom/validator/PipelineValidatorTest.java
+++ b/src/test/java/ai/preferred/venom/validator/PipelineValidatorTest.java
@@ -21,7 +21,6 @@
 import ai.preferred.venom.response.BaseResponse;
 import ai.preferred.venom.response.Response;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -36,14 +35,13 @@ public class PipelineValidatorTest {
   private final String baseUrl = "https://venom.preferred.ai";
   private final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
   private final Header[] headers = {};
-  private final HttpHost proxy = null;
 
 
   @Test
   public void testValidPipeline() {
     final int statusCode = 200;
     final byte[] content = "IPSUM".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final Validator.Status status = new PipelineValidator(StatusOkValidator.INSTANCE, EmptyContentValidator.INSTANCE)
         .isValid(request, response);
 
@@ -54,7 +52,7 @@ public void testValidPipeline() {
   public void testValidPipelineList() {
     final int statusCode = 200;
     final byte[] content = "IPSUM".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final List<Validator> validators = new LinkedList<>();
     validators.add(StatusOkValidator.INSTANCE);
     validators.add(EmptyContentValidator.INSTANCE);
@@ -68,7 +66,7 @@ public void testValidPipelineList() {
   public void testFirstInvalidPipeline() {
     final int statusCode = 400;
     final byte[] content = "IPSUM".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final Validator.Status status = new PipelineValidator(StatusOkValidator.INSTANCE, EmptyContentValidator.INSTANCE)
         .isValid(request, response);
 
@@ -79,7 +77,7 @@ public void testFirstInvalidPipeline() {
   public void testFirstInvalidPipelineList() {
     final int statusCode = 400;
     final byte[] content = "IPSUM".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final List<Validator> validators = new LinkedList<>();
     validators.add(StatusOkValidator.INSTANCE);
     validators.add(EmptyContentValidator.INSTANCE);
@@ -93,7 +91,7 @@ public void testFirstInvalidPipelineList() {
   public void testSecondInvalidPipeline() {
     final int statusCode = 200;
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final Validator.Status status = new PipelineValidator(StatusOkValidator.INSTANCE, EmptyContentValidator.INSTANCE)
         .isValid(request, response);
 
@@ -104,7 +102,7 @@ public void testSecondInvalidPipeline() {
   public void testSecondInvalidPipelineList() {
     final int statusCode = 200;
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final List<Validator> validators = new LinkedList<>();
     validators.add(StatusOkValidator.INSTANCE);
     validators.add(EmptyContentValidator.INSTANCE);
@@ -118,7 +116,7 @@ public void testSecondInvalidPipelineList() {
   public void testMultiInvalidPipeline() {
     final int statusCode = 400;
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final Validator.Status status = new PipelineValidator(StatusOkValidator.INSTANCE, EmptyContentValidator.INSTANCE)
         .isValid(request, response);
 
@@ -129,7 +127,7 @@ public void testMultiInvalidPipeline() {
   public void testMultiInvalidPipelineList() {
     final int statusCode = 400;
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final List<Validator> validators = new LinkedList<>();
     validators.add(StatusOkValidator.INSTANCE);
     validators.add(EmptyContentValidator.INSTANCE);
@@ -143,7 +141,7 @@ public void testMultiInvalidPipelineList() {
   public void testNullInPipeline() {
     final int statusCode = 400;
     final byte[] content = "".getBytes();
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     final Validator.Status status = new PipelineValidator(StatusOkValidator.INSTANCE, null)
         .isValid(request, response);
 
diff --git a/src/test/java/ai/preferred/venom/validator/StatusOkValidatorTest.java b/src/test/java/ai/preferred/venom/validator/StatusOkValidatorTest.java
index f7145c1..78330f1 100644
--- a/src/test/java/ai/preferred/venom/validator/StatusOkValidatorTest.java
+++ b/src/test/java/ai/preferred/venom/validator/StatusOkValidatorTest.java
@@ -21,7 +21,6 @@
 import ai.preferred.venom.response.BaseResponse;
 import ai.preferred.venom.response.Response;
 import org.apache.http.Header;
-import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -35,10 +34,9 @@ public class StatusOkValidatorTest {
   private final String baseUrl = "https://venom.preferred.ai";
   private final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8);
   private final Header[] headers = {};
-  private final HttpHost proxy = null;
 
   private void assertInvalid(int statusCode) {
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.INVALID_STATUS_CODE, StatusOkValidator.INSTANCE.isValid(request, response));
   }
 
@@ -51,7 +49,7 @@ public void testInvalidStatusCode() {
   @Test
   public void testValidStatusCode() {
     final int statusCode = 200;
-    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy);
+    final Response response = new BaseResponse(statusCode, baseUrl, content, contentType, headers, null);
     Assertions.assertEquals(Validator.Status.VALID, StatusOkValidator.INSTANCE.isValid(request, response));
   }
 

From 8402cec19aaf63b09faa6e4dbda980097287bb42 Mon Sep 17 00:00:00 2001
From: Lee Ween Jiann <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 14:18:36 +0800
Subject: [PATCH 13/17] Removed whitespace

---
 src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
index dd7a7bd..e908a9f 100644
--- a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
@@ -59,7 +59,6 @@ public class LazyQueueScheduler extends AbstractPriorityQueueScheduler {
    * @param handler  The default handler to use
    */
   public LazyQueueScheduler(final Iterator<Request> requests, final Handler handler) {
-
     this.requests = requests;
     this.handler = handler;
   }

From 064fd70888c1b90eb56b6bde21501786b479f401 Mon Sep 17 00:00:00 2001
From: Lee Ween Jiann <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 14:59:44 +0800
Subject: [PATCH 14/17] Updated Job

---
 .../venom/job/AbstractPriorityQueueScheduler.java    |  2 +-
 src/main/java/ai/preferred/venom/job/Job.java        |  8 +++++---
 .../preferred/venom/job/LazyQueueSchedulerTest.java  |  4 ++--
 .../venom/job/PriorityQueueSchedulerTest.java        | 12 ++++++------
 4 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
index 1a4f383..3a17033 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractPriorityQueueScheduler.java
@@ -31,7 +31,7 @@ public abstract class AbstractPriorityQueueScheduler extends AbstractQueueSchedu
    */
   protected AbstractPriorityQueueScheduler() {
     super(new PriorityBlockingQueue<>(11,
-        Comparator.comparing(o -> ((PriorityJobAttribute) o.getJobAttribute(PriorityJobAttribute.class)))));
+        Comparator.comparing(o -> (o.getJobAttribute(PriorityJobAttribute.class)))));
   }
 
   /**
diff --git a/src/main/java/ai/preferred/venom/job/Job.java b/src/main/java/ai/preferred/venom/job/Job.java
index 83de745..1c4c13e 100644
--- a/src/main/java/ai/preferred/venom/job/Job.java
+++ b/src/main/java/ai/preferred/venom/job/Job.java
@@ -146,11 +146,13 @@ public final Job addJobAttribute(final JobAttribute jobAttribute) {
    * Get the job attribute for a specific attribute class or
    * return {@code null} if not found.
    *
-   * @param clazz The class of attribute to find.
+   * @param clazz the class of attribute to find.
+   * @param <T> the class of attribute to find.
    * @return an instance of job attribute for class or null.
    */
-  public final JobAttribute getJobAttribute(final Class<? extends JobAttribute> clazz) {
-    return jobAttributeMap.get(clazz);
+  public final <T extends JobAttribute> T getJobAttribute(final Class<T> clazz) {
+    //noinspection unchecked
+    return (T) jobAttributeMap.get(clazz);
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
index 66dcc3b..28912c6 100644
--- a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
@@ -52,7 +52,7 @@ public void testIterator() {
     Assertions.assertEquals(handler, job.getHandler());
     Assertions.assertEquals(
         Priority.DEFAULT,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
@@ -82,7 +82,7 @@ public void testLazyQueue() {
     Assertions.assertNull(job.getHandler());
     Assertions.assertEquals(
         Priority.DEFAULT,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
diff --git a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
index 80cf0a1..c192c12 100644
--- a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
@@ -37,7 +37,7 @@ public void testAddRequest() {
     Assertions.assertNull(job.getHandler());
     Assertions.assertEquals(
         Priority.DEFAULT,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
@@ -59,7 +59,7 @@ public void testAddRequestHandler() {
     Assertions.assertEquals(handler, job.getHandler());
     Assertions.assertEquals(
         Priority.DEFAULT,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
@@ -82,7 +82,7 @@ public void testPriority() {
     Assertions.assertNull(job.getHandler());
     Assertions.assertEquals(
         Priority.HIGHEST,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
@@ -101,7 +101,7 @@ public void testPriorityFloor() {
     Assertions.assertNull(job.getHandler());
     Assertions.assertEquals(
         Priority.HIGH,
-        ((PriorityJobAttribute) job.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
 
     job.prepareRetry();
@@ -112,7 +112,7 @@ public void testPriorityFloor() {
     Assertions.assertNull(jobRQ.getHandler());
     Assertions.assertEquals(
         Priority.NORMAL,
-        ((PriorityJobAttribute) jobRQ.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        jobRQ.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
 
     job.prepareRetry();
@@ -123,7 +123,7 @@ public void testPriorityFloor() {
     Assertions.assertNull(jobRQRQ.getHandler());
     Assertions.assertEquals(
         Priority.NORMAL,
-        ((PriorityJobAttribute) jobRQRQ.getJobAttribute(PriorityJobAttribute.class)).getPriority()
+        jobRQRQ.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 

From 4434c49ae8ec8212363ded639e0ef36fe1039f3f Mon Sep 17 00:00:00 2001
From: Lee Ween Jiann <lwj5@hotmail.com>
Date: Sat, 27 Jul 2019 15:55:32 +0800
Subject: [PATCH 15/17] Updated Job

---
 src/main/java/ai/preferred/venom/job/Job.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/ai/preferred/venom/job/Job.java b/src/main/java/ai/preferred/venom/job/Job.java
index 1c4c13e..9336d63 100644
--- a/src/main/java/ai/preferred/venom/job/Job.java
+++ b/src/main/java/ai/preferred/venom/job/Job.java
@@ -27,6 +27,9 @@
 import java.util.Map;
 
 /**
+ * This class will be placed in a scheduler for queuing requests.
+ *
+ * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
 public class Job {
@@ -126,6 +129,7 @@ public final int getTryCount() {
    * </p>
    */
   public final void prepareRetry() {
+    LOGGER.debug("Preparing job {} - {} for next state.", Integer.toHexString(this.hashCode()), request.getUrl());
     jobAttributeMap.forEach((k, jobAttribute) -> jobAttribute.prepareRetry());
     tryCount++;
   }
@@ -147,7 +151,7 @@ public final Job addJobAttribute(final JobAttribute jobAttribute) {
    * return {@code null} if not found.
    *
    * @param clazz the class of attribute to find.
-   * @param <T> the class of attribute to find.
+   * @param <T>   the class of attribute to find.
    * @return an instance of job attribute for class or null.
    */
   public final <T extends JobAttribute> T getJobAttribute(final Class<T> clazz) {

From 139684942f0d70b903a7da1c7d77827d9e3da06c Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Mon, 29 Jul 2019 16:29:18 +0800
Subject: [PATCH 16/17] Added tests

---
 pom.xml                                       |   2 +-
 .../venom/job/AbstractQueueScheduler.java     |  59 --------
 src/main/java/ai/preferred/venom/job/Job.java |  30 ++--
 .../ai/preferred/venom/job/JobScheduler.java  |  97 ++++++++++++
 .../java/ai/preferred/venom/job/Priority.java |  14 +-
 .../ai/preferred/venom/job/Scheduler.java     |  58 ++++++++
 .../venom/job/FIFOQueueSchedulerTest.java     |  87 ++++++-----
 .../preferred/venom/job/FakeJobAttribute.java |  34 +++++
 .../preferred/venom/job/JobSchedulerTest.java |  76 ++++++++++
 .../java/ai/preferred/venom/job/JobTest.java  |  85 +++++++++++
 .../venom/job/LazyQueueSchedulerTest.java     | 114 ++++++++++----
 .../venom/job/PriorityJobAttributeTest.java   |  56 +++++++
 .../venom/job/PriorityQueueSchedulerTest.java | 140 +++++++-----------
 .../ai/preferred/venom/job/PriorityTest.java  |  38 +++++
 14 files changed, 652 insertions(+), 238 deletions(-)
 create mode 100644 src/main/java/ai/preferred/venom/job/JobScheduler.java
 create mode 100644 src/test/java/ai/preferred/venom/job/FakeJobAttribute.java
 create mode 100644 src/test/java/ai/preferred/venom/job/JobSchedulerTest.java
 create mode 100644 src/test/java/ai/preferred/venom/job/JobTest.java
 create mode 100644 src/test/java/ai/preferred/venom/job/PriorityJobAttributeTest.java
 create mode 100644 src/test/java/ai/preferred/venom/job/PriorityTest.java

diff --git a/pom.xml b/pom.xml
index 27543e5..6242ddd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
 
     <groupId>ai.preferred</groupId>
     <artifactId>venom</artifactId>
-    <version>4.1.4-SNAPSHOT</version>
+    <version>4.2.1-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <name>${project.groupId}:${project.artifactId}</name>
diff --git a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
index 9323b53..d154be0 100644
--- a/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/AbstractQueueScheduler.java
@@ -16,13 +16,7 @@
 
 package ai.preferred.venom.job;
 
-import ai.preferred.venom.Handler;
-import ai.preferred.venom.request.Request;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import javax.annotation.Nonnull;
-import javax.validation.constraints.NotNull;
 import java.util.AbstractQueue;
 import java.util.Collection;
 import java.util.Iterator;
@@ -106,57 +100,4 @@ protected final BlockingQueue<Job> getQueue() {
     return queue;
   }
 
-  /**
-   * An implementation of ai.preferred.venom.job.Scheduler using Job.
-   */
-  public static class JobScheduler implements Scheduler {
-
-    /**
-     * Logger.
-     */
-    private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduler.class);
-
-    /**
-     * The queue used for this scheduler.
-     */
-    private final QueueScheduler queueScheduler;
-
-    /**
-     * Constructs an instance of JobScheduler.
-     *
-     * @param queueScheduler an instance of BlockingQueue
-     */
-    public JobScheduler(final QueueScheduler queueScheduler) {
-      this.queueScheduler = queueScheduler;
-    }
-
-    @Override
-    public final void add(final @NotNull Request request, final @NotNull Handler handler,
-                          final JobAttribute... jobAttributes) {
-      final Job job = new Job(request, handler);
-      if (jobAttributes != null) {
-        for (JobAttribute jobAttribute : jobAttributes) {
-          job.addJobAttribute(jobAttribute);
-        }
-      }
-      queueScheduler.add(job);
-      LOGGER.debug("Job {} - {} added to queue.", job.toString(), request.getUrl());
-    }
-
-    @Override
-    public final void add(final @NotNull Request request, final JobAttribute... jobAttributes) {
-      add(request, null, jobAttributes);
-    }
-
-    @Override
-    public final void add(final Request request, final Handler handler) {
-      add(request, handler, (JobAttribute[]) null);
-    }
-
-    @Override
-    public final void add(final @NotNull Request request) {
-      add(request, null, (JobAttribute[]) null);
-    }
-
-  }
 }
diff --git a/src/main/java/ai/preferred/venom/job/Job.java b/src/main/java/ai/preferred/venom/job/Job.java
index 9336d63..a227794 100644
--- a/src/main/java/ai/preferred/venom/job/Job.java
+++ b/src/main/java/ai/preferred/venom/job/Job.java
@@ -59,16 +59,6 @@ public class Job {
    */
   private int tryCount = 1;
 
-  /**
-   * Constructs a basic job.
-   *
-   * @param request The request of this job.
-   * @param handler The handler of this job.
-   */
-  public Job(final Request request, final Handler handler) {
-    this(request, handler, (JobAttribute[]) null);
-  }
-
   /**
    * Constructs a basic job.
    *
@@ -76,7 +66,7 @@ public Job(final Request request, final Handler handler) {
    * @param handler       The handler of this job.
    * @param jobAttributes attributes to insert to the job.
    */
-  public Job(final Request request, final Handler handler, final JobAttribute... jobAttributes) {
+  public Job(final @NotNull Request request, final Handler handler, final JobAttribute... jobAttributes) {
     this.request = request;
     this.handler = handler;
     if (jobAttributes != null) {
@@ -86,6 +76,24 @@ public Job(final Request request, final Handler handler, final JobAttribute... j
     }
   }
 
+  /**
+   * Constructs a basic job.
+   *
+   * @param request The request of this job.
+   * @param handler The handler of this job.
+   */
+  public Job(final @NotNull Request request, final Handler handler) {
+    this(request, handler, (JobAttribute[]) null);
+  }
+
+  /**
+   * Constructs a basic job.
+   *
+   * @param request The request of this job.
+   */
+  public Job(final @NotNull Request request) {
+    this(request, null);
+  }
 
   /**
    * Get the request of this job.
diff --git a/src/main/java/ai/preferred/venom/job/JobScheduler.java b/src/main/java/ai/preferred/venom/job/JobScheduler.java
new file mode 100644
index 0000000..35d4c3b
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/JobScheduler.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import ai.preferred.venom.Handler;
+import ai.preferred.venom.request.Request;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * An implementation of ai.preferred.venom.job.Scheduler using Job.
+ */
+public class JobScheduler implements Scheduler {
+
+  /**
+   * Logger.
+   */
+  private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduler.class);
+
+  /**
+   * The queue used for this scheduler.
+   */
+  private final QueueScheduler queueScheduler;
+
+  /**
+   * Constructs an instance of JobScheduler.
+   *
+   * @param queueScheduler an instance of BlockingQueue
+   */
+  public JobScheduler(final QueueScheduler queueScheduler) {
+    this.queueScheduler = queueScheduler;
+  }
+
+  @Override
+  public final void add(final @NotNull Request request, final @NotNull Handler handler,
+                        final JobAttribute... jobAttributes) {
+    final Job job = new Job(request, handler);
+    if (jobAttributes != null) {
+      for (JobAttribute jobAttribute : jobAttributes) {
+        job.addJobAttribute(jobAttribute);
+      }
+    }
+    queueScheduler.add(job);
+    LOGGER.debug("Job {} - {} added to queue.", job.toString(), request.getUrl());
+  }
+
+  @Override
+  public final void add(final @NotNull Request request, final JobAttribute... jobAttributes) {
+    add(request, null, jobAttributes);
+  }
+
+  @Override
+  public final void add(final Request request, final Handler handler) {
+    add(request, handler, (JobAttribute[]) null);
+  }
+
+  @Override
+  public final void add(final @NotNull Request request) {
+    add(request, null, (JobAttribute[]) null);
+  }
+
+  @Override
+  public final void add(final @NotNull Request r, final @NotNull Handler h, final Priority p, final Priority pf) {
+    add(r, h, new PriorityJobAttribute(p, pf));
+  }
+
+  @Override
+  public final void add(final @NotNull Request r, final @NotNull Handler h, final Priority p) {
+    add(r, h, p, Priority.FLOOR);
+  }
+
+  @Override
+  public final void add(final @NotNull Request r, final Priority p, final Priority pf) {
+    add(r, null, p, pf);
+  }
+
+  @Override
+  public final void add(final @NotNull Request r, final Priority p) {
+    add(r, (Handler) null, p);
+  }
+}
diff --git a/src/main/java/ai/preferred/venom/job/Priority.java b/src/main/java/ai/preferred/venom/job/Priority.java
index 3df9bd7..f046eda 100644
--- a/src/main/java/ai/preferred/venom/job/Priority.java
+++ b/src/main/java/ai/preferred/venom/job/Priority.java
@@ -60,15 +60,6 @@ public enum Priority {
    */
   public static final Priority FLOOR = LOW;
 
-  /**
-   * Get the current priority level.
-   *
-   * @return An integer specifying current priority
-   */
-  public int getPriority() {
-    return ordinal();
-  }
-
   /**
    * Returns the priority one level below the current
    * priority if priority is higher than the specified floor or the
@@ -78,10 +69,7 @@ public int getPriority() {
    * @return Priority after downgrade.
    */
   public Priority downgrade(final Priority floor) {
-    if (this.equals(floor)) {
-      return this;
-    }
-    if (this.equals(LOWEST)) {
+    if (this.compareTo(floor) >= 0) {
       return this;
     }
     return values()[ordinal() + 1];
diff --git a/src/main/java/ai/preferred/venom/job/Scheduler.java b/src/main/java/ai/preferred/venom/job/Scheduler.java
index 7e38fce..1cf747c 100644
--- a/src/main/java/ai/preferred/venom/job/Scheduler.java
+++ b/src/main/java/ai/preferred/venom/job/Scheduler.java
@@ -74,4 +74,62 @@ public interface Scheduler {
    */
   void add(@NotNull Request request);
 
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to a minimum priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param h  handler to be used to parse the request
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  @Deprecated
+  void add(@NotNull Request r, @NotNull Handler h, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param h handler to be used to parse the request
+   * @param p initial priority of the request
+   */
+  @Deprecated
+  void add(@NotNull Request r, @NotNull Handler h, Priority p);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise, and its priority can be downgraded to a minimum
+   * priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  @Deprecated
+  void add(@NotNull Request r, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise defined, and its priority can be downgraded to the
+   * default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param p initial priority of the request
+   */
+  @Deprecated
+  void add(@NotNull Request r, Priority p);
+
 }
diff --git a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
index 36f7da4..3edc417 100644
--- a/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/FIFOQueueSchedulerTest.java
@@ -16,62 +16,71 @@
 
 package ai.preferred.venom.job;
 
-import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.VRequest;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-public class FIFOQueueSchedulerTest {
+import java.util.concurrent.TimeUnit;
 
-  @Test
-  public void testAddRequest() {
-    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+class FIFOQueueSchedulerTest {
+
+  private final String url = "https://venom.preferred.ai";
+  private final VRequest vRequest = new VRequest(url);
+  private final Job job = new Job(vRequest);
 
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
+  private FIFOQueueScheduler scheduler;
 
-    scheduler.getScheduler().add(vRequest);
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
+  @BeforeEach
+  void initEach() {
+    scheduler = new FIFOQueueScheduler();
   }
 
   @Test
-  public void testAddRequestHandler() {
-    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
-
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
-
-    final Handler handler = (request, response, schedulerH, session, worker) -> {
-
-    };
+  void testAddRequest() {
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    scheduler.getScheduler().add(vRequest, handler);
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertEquals(handler, job.getHandler());
+  @Test
+  void testPutRequest() throws InterruptedException {
+    scheduler.put(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
   }
 
   @Test
-  public void testFIFOQueue() {
-    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+  void testOfferRequest() {
+    scheduler.offer(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
-    final VRequest vRequestNeg = new VRequest(url);
+  @Test
+  void testOfferTimeoutRequest() throws InterruptedException {
+    scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGH));
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.HIGHEST));
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.DEFAULT));
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.LOW));
+  @Test
+  void testPollTimeout() throws InterruptedException {
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
+  @Test
+  void testFIFOQueue() {
+    final Job job = new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGH));
+    scheduler.add(job);
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGHEST)));
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
+
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/FakeJobAttribute.java b/src/test/java/ai/preferred/venom/job/FakeJobAttribute.java
new file mode 100644
index 0000000..362468f
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/job/FakeJobAttribute.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class FakeJobAttribute implements JobAttribute {
+
+  private final AtomicInteger count = new AtomicInteger();
+
+  @Override
+  public void prepareRetry() {
+    count.addAndGet(1);
+  }
+
+  int getCount() {
+    return count.get();
+  }
+
+}
diff --git a/src/test/java/ai/preferred/venom/job/JobSchedulerTest.java b/src/test/java/ai/preferred/venom/job/JobSchedulerTest.java
new file mode 100644
index 0000000..7b40190
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/job/JobSchedulerTest.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import ai.preferred.venom.Handler;
+import ai.preferred.venom.request.VRequest;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class JobSchedulerTest {
+
+  private final String url = "https://venom.preferred.ai";
+  private final VRequest vRequest = new VRequest(url);
+  private final Handler handler = (request, response, scheduler, session, worker) -> {
+
+  };
+
+  @Test
+  void testAddRequest() {
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+    scheduler.getScheduler().add(vRequest);
+    final Job job = scheduler.poll();
+    Assertions.assertNotNull(job);
+    Assertions.assertEquals(vRequest, job.getRequest());
+    Assertions.assertNull(job.getHandler());
+  }
+
+  @Test
+  void testAddRequestHandler() {
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+    scheduler.getScheduler().add(vRequest, handler);
+    final Job job = scheduler.poll();
+    Assertions.assertNotNull(job);
+    Assertions.assertEquals(vRequest, job.getRequest());
+    Assertions.assertEquals(handler, job.getHandler());
+  }
+
+  @Test
+  void testAddRequestJobAttribute() {
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+    final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute();
+    scheduler.getScheduler().add(vRequest, priorityJobAttribute);
+    final Job job = scheduler.poll();
+    Assertions.assertNotNull(job);
+    Assertions.assertEquals(vRequest, job.getRequest());
+    Assertions.assertNull(job.getHandler());
+    Assertions.assertEquals(priorityJobAttribute, job.getJobAttribute(priorityJobAttribute.getClass()));
+  }
+
+  @Test
+  void testAddRequestHandlerJobAttribute() {
+    final FIFOQueueScheduler scheduler = new FIFOQueueScheduler();
+    final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute();
+    scheduler.getScheduler().add(vRequest, handler, priorityJobAttribute);
+    final Job job = scheduler.poll();
+    Assertions.assertNotNull(job);
+    Assertions.assertEquals(vRequest, job.getRequest());
+    Assertions.assertEquals(handler, job.getHandler());
+    Assertions.assertEquals(priorityJobAttribute, job.getJobAttribute(priorityJobAttribute.getClass()));
+  }
+
+}
diff --git a/src/test/java/ai/preferred/venom/job/JobTest.java b/src/test/java/ai/preferred/venom/job/JobTest.java
new file mode 100644
index 0000000..b31c4dc
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/job/JobTest.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import ai.preferred.venom.Handler;
+import ai.preferred.venom.request.VRequest;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class JobTest {
+
+
+  @Test
+  void testJob() {
+    final String url = "https://venom.preferred.ai";
+    final VRequest vRequest = new VRequest(url);
+
+    final Handler handler = (request, response, scheduler, session, worker) -> {
+
+    };
+
+    final FakeJobAttribute fakeJobAttribute = new FakeJobAttribute();
+
+    final Job job = new Job(vRequest, handler, fakeJobAttribute);
+    Assertions.assertEquals(vRequest, job.getRequest());
+    Assertions.assertEquals(handler, job.getHandler());
+
+    final JobAttribute jobAttribute = job.getJobAttribute(fakeJobAttribute.getClass());
+    Assertions.assertEquals(fakeJobAttribute, jobAttribute);
+  }
+
+  @Test
+  void testJobTryCount() {
+    final String url = "https://venom.preferred.ai";
+    final VRequest vRequest = new VRequest(url);
+
+    final Job job = new Job(vRequest, null);
+    Assertions.assertEquals(1, job.getTryCount());
+
+    job.prepareRetry();
+    Assertions.assertEquals(2, job.getTryCount());
+  }
+
+  @Test
+  void testJobPrepareRetry() {
+    final String url = "https://venom.preferred.ai";
+    final VRequest vRequest = new VRequest(url);
+
+    final FakeJobAttribute fakeJobAttribute = new FakeJobAttribute();
+    final Job job = new Job(vRequest, null, fakeJobAttribute);
+    Assertions.assertEquals(0, fakeJobAttribute.getCount());
+
+    job.prepareRetry();
+    Assertions.assertEquals(1, fakeJobAttribute.getCount());
+  }
+
+  @Test
+  void testAddJobAttributes() {
+    final String url = "https://venom.preferred.ai";
+    final VRequest vRequest = new VRequest(url);
+
+    final Job job = new Job(vRequest, null);
+
+    final FakeJobAttribute fakeJobAttribute = new FakeJobAttribute();
+
+    job.addJobAttribute(fakeJobAttribute);
+    final JobAttribute jobAttribute = job.getJobAttribute(fakeJobAttribute.getClass());
+    Assertions.assertEquals(fakeJobAttribute, jobAttribute);
+  }
+
+}
diff --git a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
index 28912c6..8c016c3 100644
--- a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
@@ -20,20 +20,73 @@
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.request.VRequest;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
-public class LazyQueueSchedulerTest {
+class LazyQueueSchedulerTest {
+
+  private final String url = "https://venom.preferred.ai";
+  private final VRequest vRequest = new VRequest(url);
+  private final Job job = new Job(vRequest);
+
+  private LazyQueueScheduler scheduler;
+
+  @BeforeEach
+  void initEach() {
+    scheduler = new LazyQueueScheduler(null);
+  }
 
   @Test
-  public void testIterator() {
-    final List<Request> requests = new ArrayList<>();
+  void testAddRequest() {
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
+
+  @Test
+  void testPutRequest() throws InterruptedException {
+    scheduler.put(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
+
+  @Test
+  void testOfferRequest() {
+    scheduler.offer(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
+
+  @Test
+  void testOfferTimeoutRequest() throws InterruptedException {
+    scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
+
+  @Test
+  void testPollTimeout() throws InterruptedException {
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    final String url = "https://venom.preferred.ai";
+  @Test
+  void testIterator() {
+    final List<Request> requests = new ArrayList<>();
     final VRequest vRequestNeg = new VRequest(url);
-    final VRequest vRequest = new VRequest(url);
 
     requests.add(vRequest);
     requests.add(vRequestNeg);
@@ -46,21 +99,20 @@ public void testIterator() {
 
     final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
 
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertEquals(handler, job.getHandler());
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(vRequest, pollJob.getRequest());
+    Assertions.assertEquals(handler, pollJob.getHandler());
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
     Assertions.assertEquals(
         Priority.DEFAULT,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
+        pollJob.getJobAttribute(PriorityJobAttribute.class).getPriority()
     );
   }
 
   @Test
-  public void testLazyQueue() {
+  void testLazyQueue() {
     final List<Request> requests = new ArrayList<>();
-
-    final String url = "https://venom.preferred.ai";
     final VRequest vRequestNeg = new VRequest(url);
 
     requests.add(vRequestNeg);
@@ -68,22 +120,30 @@ public void testLazyQueue() {
     requests.add(vRequestNeg);
     requests.add(vRequestNeg);
 
-    final Handler handler = (request, response, schedulerH, session, worker) -> {
-
-    };
+    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator());
+    final Job job = new Job(vRequest);
 
-    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
-    final VRequest vRequest = new VRequest(url);
-    scheduler.getScheduler().add(vRequest);
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(
-        Priority.DEFAULT,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+  @Test
+  void testIsEmpty() {
+    final List<Request> requests = new ArrayList<>();
+    requests.add(vRequest);
+    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator());
+
+    scheduler.add(job);
+    Assertions.assertFalse(scheduler.isEmpty());
+    scheduler.poll();
+    Assertions.assertFalse(scheduler.isEmpty());
+    scheduler.poll();
+    Assertions.assertTrue(scheduler.isEmpty());
+    scheduler.add(job);
+    Assertions.assertFalse(scheduler.isEmpty());
+    scheduler.poll();
+    Assertions.assertTrue(scheduler.isEmpty());
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/PriorityJobAttributeTest.java b/src/test/java/ai/preferred/venom/job/PriorityJobAttributeTest.java
new file mode 100644
index 0000000..72ee251
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/job/PriorityJobAttributeTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class PriorityJobAttributeTest {
+
+  @Test
+  void testPriority() {
+    final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute(Priority.HIGHEST);
+    Assertions.assertEquals(Priority.HIGHEST, priorityJobAttribute.getPriority());
+
+    final PriorityJobAttribute priorityJobAttributeDefault = new PriorityJobAttribute();
+    Assertions.assertEquals(Priority.DEFAULT, priorityJobAttributeDefault.getPriority());
+  }
+
+  @Test
+  void testPriorityFloor() {
+    final PriorityJobAttribute priorityJobAttribute = new PriorityJobAttribute(Priority.HIGH);
+    Assertions.assertEquals(Priority.HIGH, priorityJobAttribute.getPriority());
+    priorityJobAttribute.prepareRetry();
+    Assertions.assertEquals(Priority.NORMAL, priorityJobAttribute.getPriority());
+    priorityJobAttribute.prepareRetry();
+    Assertions.assertEquals(Priority.LOW, priorityJobAttribute.getPriority());
+    priorityJobAttribute.prepareRetry();
+    Assertions.assertEquals(Priority.LOW, priorityJobAttribute.getPriority());
+  }
+
+  @Test
+  void testCompare() {
+    final PriorityJobAttribute priorityJobAttributeHigh = new PriorityJobAttribute(Priority.HIGH);
+    final PriorityJobAttribute priorityJobAttributeLow = new PriorityJobAttribute(Priority.LOW);
+    Assertions.assertTrue(priorityJobAttributeHigh.compareTo(priorityJobAttributeLow) < 0);
+    Assertions.assertTrue(priorityJobAttributeLow.compareTo(priorityJobAttributeHigh) > 0);
+
+    final PriorityJobAttribute priorityJobAttributeLow2 = new PriorityJobAttribute(Priority.LOW);
+    Assertions.assertEquals(0, priorityJobAttributeLow2.compareTo(priorityJobAttributeLow));
+  }
+
+}
diff --git a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
index c192c12..369d161 100644
--- a/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/PriorityQueueSchedulerTest.java
@@ -16,115 +16,79 @@
 
 package ai.preferred.venom.job;
 
-import ai.preferred.venom.Handler;
 import ai.preferred.venom.request.VRequest;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-public class PriorityQueueSchedulerTest {
+import java.util.concurrent.TimeUnit;
 
-  @Test
-  public void testAddRequest() {
-    final PriorityQueueScheduler scheduler = new PriorityQueueScheduler();
+class PriorityQueueSchedulerTest {
+
+  private final String url = "https://venom.preferred.ai";
+  private final VRequest vRequest = new VRequest(url);
+  private final Job job = new Job(vRequest);
 
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
+  private PriorityQueueScheduler scheduler;
 
-    scheduler.getScheduler().add(vRequest);
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(
-        Priority.DEFAULT,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+  @BeforeEach
+  void initEach() {
+    scheduler = new PriorityQueueScheduler();
   }
 
   @Test
-  public void testAddRequestHandler() {
-    final PriorityQueueScheduler scheduler = new PriorityQueueScheduler();
-
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
-
-    final Handler handler = (request, response, schedulerH, session, worker) -> {
-
-    };
-
-    scheduler.getScheduler().add(vRequest, handler);
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertEquals(handler, job.getHandler());
-    Assertions.assertEquals(
-        Priority.DEFAULT,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+  void testAddRequest() {
+    scheduler.add(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
   }
 
   @Test
-  public void testPriority() {
-    final PriorityQueueScheduler scheduler = new PriorityQueueScheduler();
-
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
-    final VRequest vRequestNeg = new VRequest(url);
-
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.HIGH));
-    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGHEST));
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.DEFAULT));
-    scheduler.getScheduler().add(vRequestNeg, new PriorityJobAttribute(Priority.LOW));
-
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(
-        Priority.HIGHEST,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+  void testPutRequest() throws InterruptedException {
+    scheduler.put(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
   }
 
   @Test
-  public void testPriorityFloor() {
-    final PriorityQueueScheduler scheduler = new PriorityQueueScheduler();
-
-    final String url = "https://venom.preferred.ai";
-    final VRequest vRequest = new VRequest(url);
-
-    scheduler.getScheduler().add(vRequest, new PriorityJobAttribute(Priority.HIGH, Priority.NORMAL));
+  void testOfferRequest() {
+    scheduler.offer(job);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
 
-    final Job job = scheduler.poll();
-    Assertions.assertNotNull(job);
-    Assertions.assertEquals(vRequest, job.getRequest());
-    Assertions.assertNull(job.getHandler());
-    Assertions.assertEquals(
-        Priority.HIGH,
-        job.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+  @Test
+  void testOfferTimeoutRequest() throws InterruptedException {
+    scheduler.offer(job, 1L, TimeUnit.NANOSECONDS);
+    final Job pollJob = scheduler.poll();
+    Assertions.assertNotNull(pollJob);
+    Assertions.assertEquals(job, pollJob);
+    Assertions.assertNotNull(pollJob.getJobAttribute(PriorityJobAttribute.class));
+  }
 
-    job.prepareRetry();
+  @Test
+  void testPollTimeout() throws InterruptedException {
     scheduler.add(job);
-    final Job jobRQ = scheduler.poll();
-    Assertions.assertNotNull(jobRQ);
-    Assertions.assertEquals(vRequest, jobRQ.getRequest());
-    Assertions.assertNull(jobRQ.getHandler());
-    Assertions.assertEquals(
-        Priority.NORMAL,
-        jobRQ.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+    final Job pollJob = scheduler.poll(1L, TimeUnit.NANOSECONDS);
+    Assertions.assertEquals(job, pollJob);
+  }
 
-    job.prepareRetry();
+  @Test
+  void testPriorityQueue() {
+    final Job job = new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGHEST));
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.HIGH)));
     scheduler.add(job);
-    final Job jobRQRQ = scheduler.poll();
-    Assertions.assertNotNull(jobRQRQ);
-    Assertions.assertEquals(vRequest, jobRQRQ.getRequest());
-    Assertions.assertNull(jobRQRQ.getHandler());
-    Assertions.assertEquals(
-        Priority.NORMAL,
-        jobRQRQ.getJobAttribute(PriorityJobAttribute.class).getPriority()
-    );
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.DEFAULT)));
+    scheduler.add(new Job(vRequest, null, new PriorityJobAttribute(Priority.LOW)));
+
+    final Job pollJob = scheduler.poll();
+    Assertions.assertEquals(pollJob, job);
   }
 
 }
diff --git a/src/test/java/ai/preferred/venom/job/PriorityTest.java b/src/test/java/ai/preferred/venom/job/PriorityTest.java
new file mode 100644
index 0000000..6e416fe
--- /dev/null
+++ b/src/test/java/ai/preferred/venom/job/PriorityTest.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class PriorityTest {
+
+  @Test
+  void testDowngrade() {
+    Assertions.assertEquals(Priority.FLOOR, Priority.FLOOR.downgrade());
+    Assertions.assertEquals(Priority.HIGH, Priority.HIGHEST.downgrade());
+  }
+
+  @Test
+  void testDowngradeFloor() {
+    Assertions.assertEquals(Priority.HIGH, Priority.HIGHEST.downgrade(Priority.HIGH));
+    Assertions.assertEquals(Priority.HIGH, Priority.HIGH.downgrade(Priority.HIGH));
+    Assertions.assertEquals(Priority.LOWEST, Priority.LOW.downgrade(Priority.LOWEST));
+    Assertions.assertEquals(Priority.LOWEST, Priority.LOWEST.downgrade(Priority.LOWEST));
+  }
+
+}

From 9f17723094de3fba9065da951540a551c0dc57b1 Mon Sep 17 00:00:00 2001
From: Ween Jiann Lee <lwj5@hotmail.com>
Date: Mon, 29 Jul 2019 17:43:56 +0800
Subject: [PATCH 17/17] Create backward compatibility

---
 ...r.java => LazyPriorityQueueScheduler.java} |   6 +-
 .../ai/preferred/venom/job/LazyScheduler.java | 198 ++++++++++++++++++
 .../java/ai/preferred/venom/CrawlerTest.java  |   4 +-
 ...va => LazyPriorityQueueSchedulerTest.java} |  12 +-
 4 files changed, 209 insertions(+), 11 deletions(-)
 rename src/main/java/ai/preferred/venom/job/{LazyQueueScheduler.java => LazyPriorityQueueScheduler.java} (91%)
 create mode 100644 src/main/java/ai/preferred/venom/job/LazyScheduler.java
 rename src/test/java/ai/preferred/venom/job/{LazyQueueSchedulerTest.java => LazyPriorityQueueSchedulerTest.java} (90%)

diff --git a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java b/src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java
similarity index 91%
rename from src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
rename to src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java
index e908a9f..a496656 100644
--- a/src/main/java/ai/preferred/venom/job/LazyQueueScheduler.java
+++ b/src/main/java/ai/preferred/venom/job/LazyPriorityQueueScheduler.java
@@ -35,7 +35,7 @@
  * @author Maksim Tkachenko
  * @author Ween Jiann Lee
  */
-public class LazyQueueScheduler extends AbstractPriorityQueueScheduler {
+public class LazyPriorityQueueScheduler extends AbstractPriorityQueueScheduler {
 
   /**
    * An object to synchronise upon.
@@ -58,7 +58,7 @@ public class LazyQueueScheduler extends AbstractPriorityQueueScheduler {
    * @param requests An iterator to obtain requests
    * @param handler  The default handler to use
    */
-  public LazyQueueScheduler(final Iterator<Request> requests, final Handler handler) {
+  public LazyPriorityQueueScheduler(final Iterator<Request> requests, final Handler handler) {
     this.requests = requests;
     this.handler = handler;
   }
@@ -68,7 +68,7 @@ public LazyQueueScheduler(final Iterator<Request> requests, final Handler handle
    *
    * @param requests An iterator to obtain requests
    */
-  public LazyQueueScheduler(final Iterator<Request> requests) {
+  public LazyPriorityQueueScheduler(final Iterator<Request> requests) {
     this(requests, null);
   }
 
diff --git a/src/main/java/ai/preferred/venom/job/LazyScheduler.java b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
new file mode 100644
index 0000000..0c30010
--- /dev/null
+++ b/src/main/java/ai/preferred/venom/job/LazyScheduler.java
@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2019 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom.job;
+
+import ai.preferred.venom.Handler;
+import ai.preferred.venom.request.Request;
+
+import javax.annotation.Nonnull;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Deprecated
+ */
+@Deprecated
+public class LazyScheduler implements QueueScheduler {
+
+  /**
+   * Link
+   */
+  private final LazyPriorityQueueScheduler lazyPriorityQueueScheduler;
+
+  /**
+   * Constructs an instance of lazy scheduler with a default handler.
+   *
+   * @param requests An iterator to obtain requests
+   * @param handler  The default handler to use
+   */
+  public LazyScheduler(final Iterator<Request> requests, final Handler handler) {
+    lazyPriorityQueueScheduler = new LazyPriorityQueueScheduler(requests, handler);
+  }
+
+  /**
+   * Constructs an instance of lazy scheduler without a default handler.
+   *
+   * @param requests An iterator to obtain requests
+   */
+  public LazyScheduler(final Iterator<Request> requests) {
+    lazyPriorityQueueScheduler = new LazyPriorityQueueScheduler(requests);
+  }
+
+  @Override
+  public final Scheduler getScheduler() {
+    return lazyPriorityQueueScheduler.getScheduler();
+  }
+
+  @Override
+  public final boolean add(@Nonnull final Job job) {
+    return lazyPriorityQueueScheduler.add(job);
+  }
+
+  @Override
+  public final boolean offer(@Nonnull final Job job) {
+    return lazyPriorityQueueScheduler.offer(job);
+  }
+
+  @Override
+  public final Job remove() {
+    return lazyPriorityQueueScheduler.remove();
+  }
+
+  @Override
+  public final Job poll() {
+    return lazyPriorityQueueScheduler.poll();
+  }
+
+  @Override
+  public final Job element() {
+    return lazyPriorityQueueScheduler.element();
+  }
+
+  @Override
+  public final Job peek() {
+    return lazyPriorityQueueScheduler.peek();
+  }
+
+  @Override
+  public final void put(@Nonnull final Job job) throws InterruptedException {
+    lazyPriorityQueueScheduler.put(job);
+  }
+
+  @Override
+  public final boolean offer(final Job job, final long timeout, @Nonnull final TimeUnit unit) throws InterruptedException {
+    return lazyPriorityQueueScheduler.offer(job, timeout, unit);
+  }
+
+  @Override
+  public final Job take() throws InterruptedException {
+    return lazyPriorityQueueScheduler.take();
+  }
+
+  @Override
+  public final Job poll(final long timeout, @Nonnull final TimeUnit unit) throws InterruptedException {
+    return lazyPriorityQueueScheduler.poll(timeout, unit);
+  }
+
+  @Override
+  public final int remainingCapacity() {
+    return lazyPriorityQueueScheduler.remainingCapacity();
+  }
+
+  @Override
+  public final boolean remove(final Object o) {
+    return lazyPriorityQueueScheduler.remove(o);
+  }
+
+  @Override
+  public final boolean containsAll(@Nonnull final Collection<?> c) {
+    return lazyPriorityQueueScheduler.containsAll(c);
+  }
+
+  @Override
+  public final boolean addAll(@Nonnull final Collection<? extends Job> c) {
+    return lazyPriorityQueueScheduler.addAll(c);
+  }
+
+  @Override
+  public final boolean removeAll(@Nonnull final Collection<?> c) {
+    return lazyPriorityQueueScheduler.removeAll(c);
+  }
+
+  @Override
+  public final boolean retainAll(@Nonnull final Collection<?> c) {
+    return lazyPriorityQueueScheduler.retainAll(c);
+  }
+
+  @Override
+  public final void clear() {
+    lazyPriorityQueueScheduler.clear();
+  }
+
+  @SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
+  @Override
+  public final boolean equals(final Object o) {
+    return lazyPriorityQueueScheduler.equals(o);
+  }
+
+  @Override
+  public final int hashCode() {
+    return lazyPriorityQueueScheduler.hashCode();
+  }
+
+  @Override
+  public final int size() {
+    return lazyPriorityQueueScheduler.size();
+  }
+
+  @Override
+  public final boolean isEmpty() {
+    return lazyPriorityQueueScheduler.isEmpty();
+  }
+
+  @Override
+  public final boolean contains(final Object o) {
+    return lazyPriorityQueueScheduler.contains(o);
+  }
+
+  @Override
+  public final Iterator<Job> iterator() {
+    return lazyPriorityQueueScheduler.iterator();
+  }
+
+  @Override
+  public final Object[] toArray() {
+    return lazyPriorityQueueScheduler.toArray();
+  }
+
+  @SuppressWarnings("SuspiciousToArrayCall")
+  @Override
+  public final <T> T[] toArray(@Nonnull final T[] a) {
+    return lazyPriorityQueueScheduler.toArray(a);
+  }
+
+  @Override
+  public final int drainTo(@Nonnull final Collection<? super Job> c) {
+    return lazyPriorityQueueScheduler.drainTo(c);
+  }
+
+  @Override
+  public final int drainTo(@Nonnull final Collection<? super Job> c, final int maxElements) {
+    return lazyPriorityQueueScheduler.drainTo(c, maxElements);
+  }
+}
diff --git a/src/test/java/ai/preferred/venom/CrawlerTest.java b/src/test/java/ai/preferred/venom/CrawlerTest.java
index e7bcf16..a85f22f 100644
--- a/src/test/java/ai/preferred/venom/CrawlerTest.java
+++ b/src/test/java/ai/preferred/venom/CrawlerTest.java
@@ -19,7 +19,7 @@
 import ai.preferred.venom.fetcher.FakeFetcher;
 import ai.preferred.venom.fetcher.Fetcher;
 import ai.preferred.venom.job.FIFOQueueScheduler;
-import ai.preferred.venom.job.LazyQueueScheduler;
+import ai.preferred.venom.job.LazyPriorityQueueScheduler;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.request.VRequest;
 import org.apache.http.HttpHost;
@@ -239,7 +239,7 @@ public void testLazySchedulerIntegration() throws Exception {
         .setMaxConnections(1)
         .setPropRetainProxy(0.2)
         .setMaxTries(5)
-        .setScheduler(new LazyQueueScheduler(requests.iterator(), handler))
+        .setScheduler(new LazyPriorityQueueScheduler(requests.iterator(), handler))
         .setSleepScheduler(new SleepScheduler(0))
         .build()
         .start()) {
diff --git a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java b/src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java
similarity index 90%
rename from src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
rename to src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java
index 8c016c3..843f4dc 100644
--- a/src/test/java/ai/preferred/venom/job/LazyQueueSchedulerTest.java
+++ b/src/test/java/ai/preferred/venom/job/LazyPriorityQueueSchedulerTest.java
@@ -27,17 +27,17 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-class LazyQueueSchedulerTest {
+class LazyPriorityQueueSchedulerTest {
 
   private final String url = "https://venom.preferred.ai";
   private final VRequest vRequest = new VRequest(url);
   private final Job job = new Job(vRequest);
 
-  private LazyQueueScheduler scheduler;
+  private LazyPriorityQueueScheduler scheduler;
 
   @BeforeEach
   void initEach() {
-    scheduler = new LazyQueueScheduler(null);
+    scheduler = new LazyPriorityQueueScheduler(null);
   }
 
   @Test
@@ -97,7 +97,7 @@ void testIterator() {
 
     };
 
-    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator(), handler);
+    final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator(), handler);
 
     final Job pollJob = scheduler.poll();
     Assertions.assertNotNull(pollJob);
@@ -120,7 +120,7 @@ void testLazyQueue() {
     requests.add(vRequestNeg);
     requests.add(vRequestNeg);
 
-    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator());
+    final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator());
     final Job job = new Job(vRequest);
 
     scheduler.add(job);
@@ -132,7 +132,7 @@ void testLazyQueue() {
   void testIsEmpty() {
     final List<Request> requests = new ArrayList<>();
     requests.add(vRequest);
-    final LazyQueueScheduler scheduler = new LazyQueueScheduler(requests.iterator());
+    final LazyPriorityQueueScheduler scheduler = new LazyPriorityQueueScheduler(requests.iterator());
 
     scheduler.add(job);
     Assertions.assertFalse(scheduler.isEmpty());