Separated Scheduler interface

src/main/java/ai/preferred/venom/Crawler.java

@@ -17,7 +17,6 @@
 package ai.preferred.venom;
 
 import ai.preferred.venom.fetcher.*;
-import ai.preferred.venom.job.AbstractQueueScheduler;
 import ai.preferred.venom.job.Job;
 import ai.preferred.venom.job.PriorityQueueScheduler;
 import ai.preferred.venom.job.Scheduler;
@@ -89,7 +88,7 @@ public final class Crawler implements Interruptible {
    * The scheduler used.
    */
   @NotNull
-  private final AbstractQueueScheduler scheduler;
+  private final Scheduler<? extends Job> scheduler;
 
   /**
    * The maximum number of simultaneous connections.
@@ -293,8 +292,8 @@ private void run() {
    *
    * @return the instance of scheduler used.
    */
-  public Scheduler getScheduler() {
-    return scheduler;
+  public ai.preferred.venom.Scheduler getScheduler() {
+    return scheduler.getScheduler();
   }
 
   /**
@@ -467,7 +466,7 @@ public static final class Builder {
     /**
      * The scheduler used.
      */
-    private AbstractQueueScheduler scheduler;
+    private Scheduler<? extends Job> scheduler;
 
     /**
      * The sleep scheduler used.
@@ -550,7 +549,7 @@ public Builder setWorkerManager(final @NotNull WorkerManager workerManager) {
      * @param scheduler scheduler to be used.
      * @return this
      */
-    public Builder setScheduler(final @NotNull AbstractQueueScheduler scheduler) {
+    public Builder setScheduler(final @NotNull Scheduler<? extends Job> scheduler) {
       this.scheduler = scheduler;
       return this;
     }
@@ -674,13 +673,13 @@ public void completed(final Request request, final Response response) {
       crawler.threadPool.execute(() -> {
         try {
           if (job.getHandler() != null) {
-            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.scheduler, crawler.session,
-                crawler.workerManager.getWorker());
+            job.getHandler().handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+                crawler.session, crawler.workerManager.getWorker());
           } else if (crawler.router != null) {
             final Handler routedHandler = crawler.router.getHandler(job.getRequest());
             if (routedHandler != null) {
-              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.scheduler, crawler.session,
-                  crawler.workerManager.getWorker());
+              routedHandler.handle(job.getRequest(), new VResponse(response), crawler.scheduler.getScheduler(),
+                  crawler.session, crawler.workerManager.getWorker());
             }
           } else {
             LOGGER.error("No handler to handle request {}.", job.getRequest().getUrl());

src/main/java/ai/preferred/venom/FatalHandlerException.java

@@ -3,7 +3,7 @@
 /**
  * This class defines fatal runtime exception for {@link Handler}.
  * If {@link Handler#handle(ai.preferred.venom.request.Request,
- * ai.preferred.venom.response.VResponse, ai.preferred.venom.job.Scheduler,
+ * ai.preferred.venom.response.VResponse, ai.preferred.venom.Scheduler,
  * Session, Worker)} encounters unexpected situation, it can signal
  * {@link Crawler} to stop execution by throwing this exception.
  *

src/main/java/ai/preferred/venom/Handler.java

@@ -16,7 +16,6 @@
 
 package ai.preferred.venom;
 
-import ai.preferred.venom.job.Scheduler;
 import ai.preferred.venom.request.Request;
 import ai.preferred.venom.response.VResponse;
 

src/main/java/ai/preferred/venom/Scheduler.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 Preferred.AI
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.preferred.venom;
+
+import ai.preferred.venom.job.Priority;
+import ai.preferred.venom.request.Request;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * This interface represents only the adding part a scheduler.
+ *
+ * @author Maksim Tkachenko
+ * @author Ween Jiann Lee
+ */
+public interface Scheduler {
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to a minimum priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param h  handler to be used to parse the request
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  void add(@NotNull Request r, @NotNull Handler h, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * its priority can be downgraded to the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param h handler to be used to parse the request
+   * @param p initial priority of the request
+   */
+  void add(@NotNull Request r, @NotNull Handler h, Priority p);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by the handler specified, and
+   * it's initialised with default priority that can be downgraded to
+   * the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param h handler to be used to parse the request
+   */
+  void add(@NotNull Request r, @NotNull Handler h);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise, and its priority can be downgraded to a minimum
+   * priority specified.
+   * </p>
+   *
+   * @param r  request to fetch when dequeued
+   * @param p  initial priority of the request
+   * @param pf the minimum (floor) priority of this request
+   */
+  void add(@NotNull Request r, Priority p, Priority pf);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise defined, and its priority can be downgraded to the
+   * default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   * @param p initial priority of the request
+   */
+  void add(@NotNull Request r, Priority p);
+
+  /**
+   * Adds a request to the queue.
+   * <p>
+   * This request would be parsed by a handler defined in Router
+   * or otherwise defined, and it's initialised with default priority
+   * that can be downgraded to the default minimum priority.
+   * </p>
+   *
+   * @param r request to fetch when dequeued
+   */
+  void add(@NotNull Request r);
+
+}

src/main/java/ai/preferred/venom/fetcher/AsyncFetcher.java

@@ -133,7 +133,7 @@ public final class AsyncFetcher implements Fetcher {
   private final boolean compressed;
 
   /**
-   * Constructs an instance of async fetcher.
+   * Constructs an instance of AsyncFetcher.
    *
    * @param builder An instance of builder
    */
@@ -186,7 +186,7 @@ private AsyncFetcher(final Builder builder) {
   }
 
   /**
-   * Create an instance of async fetcher with default options.
+   * Create an instance of AsyncFetcher with default options.
    *
    * @return A new instance of async fetcher
    */
@@ -205,10 +205,10 @@ public static Builder builder() {
 
   /**
    * Check if request is an instance of http fetcher request and return it
-   * if true, otherwise wrap it with http fetcher request and return that.
+   * if true, otherwise wrap it with HttpFetcherRequest and return that.
    *
    * @param request An instance of request
-   * @return An instance of http fetcher request
+   * @return An instance of HttpFetcherRequest
    */
   private HttpFetcherRequest normalizeRequest(final Request request) {
     if (request instanceof HttpFetcherRequest) {
@@ -221,7 +221,7 @@ private HttpFetcherRequest normalizeRequest(final Request request) {
    * Prepare fetcher request by prepending headers and set appropriate proxy.
    *
    * @param request An instance of request
-   * @return An instance of http fetcher request
+   * @return An instance of HttpFetcherRequest
    */
   private HttpFetcherRequest prepareFetcherRequest(final Request request) {
     HttpFetcherRequest httpFetcherRequest = normalizeRequest(request);