Added tests for Storage

src/main/java/ai/preferred/venom/ThreadedWorkerManager.java

@@ -49,7 +49,7 @@ public class ThreadedWorkerManager implements WorkerManager {
    *
    * @param executor An executor service
    */
-  public ThreadedWorkerManager(final ExecutorService executor) {
+  public ThreadedWorkerManager(@Nullable final ExecutorService executor) {
     this.executor = executor;
     if (executor instanceof ForkJoinPool || executor == null) {
       this.worker = new ForkJoinWorker();

src/main/java/ai/preferred/venom/fetcher/Callback.java

@@ -32,17 +32,17 @@ public interface Callback {
   Callback EMPTY_CALLBACK = new Callback() {
     @Override
     public void completed(final @NotNull Request request, final @NotNull Response response) {
-      // Do nothing
+      // do nothing
     }
 
     @Override
     public void failed(final @NotNull Request request, final @NotNull Exception ex) {
-      // Do nothing
+      // do nothing
     }
 
     @Override
     public void cancelled(final @NotNull Request request) {
-      // Do nothing
+      // do nothing
     }
   };
 

src/main/java/ai/preferred/venom/fetcher/MysqlFetcher.java → src/main/java/ai/preferred/venom/fetcher/StorageFetcher.java

@@ -16,8 +16,8 @@
 
 package ai.preferred.venom.fetcher;
 
-import ai.preferred.venom.request.MysqlFetcherRequest;
 import ai.preferred.venom.request.Request;
+import ai.preferred.venom.request.StorageFetcherRequest;
 import ai.preferred.venom.request.Unwrappable;
 import ai.preferred.venom.response.Response;
 import ai.preferred.venom.response.StorageResponse;
@@ -29,39 +29,29 @@
 import ai.preferred.venom.validator.PipelineValidator;
 import ai.preferred.venom.validator.StatusOkValidator;
 import ai.preferred.venom.validator.Validator;
-import org.apache.http.ParseException;
 import org.apache.http.concurrent.BasicFuture;
 import org.apache.http.concurrent.FutureCallback;
-import org.apache.http.entity.ContentType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.validation.constraints.NotNull;
-import java.io.IOException;
-import java.net.MalformedURLException;
 import java.net.URISyntaxException;
-import java.nio.charset.UnsupportedCharsetException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.Future;
 
 /**
- * This class holds the implementation to provide how items are fetched from a MySQL database,
+ * This class holds the implementation to provide how items are fetched from a database,
  * to validate the item and to store it if specified.
  *
  * @author Ween Jiann Lee
  */
-public final class MysqlFetcher implements Fetcher {
+public final class StorageFetcher implements Fetcher {
 
   /**
    * Logger.
    */
-  private static final Logger LOGGER = LoggerFactory.getLogger(MysqlFetcher.class);
-
-  /**
-   * Default content type of response if not given.
-   */
-  private static final ContentType DEFAULT_CONTENT_TYPE = ContentType.APPLICATION_OCTET_STREAM;
+  private static final Logger LOGGER = LoggerFactory.getLogger(StorageFetcher.class);
 
   /**
    * The file manager used to store raw responses.
@@ -79,11 +69,11 @@ public final class MysqlFetcher implements Fetcher {
   private final Map<String, String> headers;
 
   /**
-   * Constructs an instance of MySQL fetcher.
+   * Constructs an instance of StorageFetcher.
    *
    * @param builder An instance of builder
    */
-  private MysqlFetcher(final Builder builder) {
+  private StorageFetcher(final Builder builder) {
     this.fileManager = builder.fileManager;
     this.validator = builder.validator;
     this.headers = builder.headers;
@@ -100,34 +90,17 @@ public static Builder builder(final FileManager fileManager) {
   }
 
   /**
-   * Check if request is an instance of MySQL fetcher request and return it
-   * if true, otherwise wrap it with MySQL fetcher request and return that.
+   * Check if request is an instance of StorageFetcher request and return it
+   * if true, otherwise wrap it with StorageFetcherRequest and return that.
    *
    * @param request An instance of request
-   * @return An instance of MySQL fetcher request
+   * @return An instance of StorageFetcherRequest
    */
-  private MysqlFetcherRequest normalize(final Request request) {
-    if (request instanceof MysqlFetcherRequest) {
-      return (MysqlFetcherRequest) request;
+  private StorageFetcherRequest normalize(final Request request) {
+    if (request instanceof StorageFetcherRequest) {
+      return (StorageFetcherRequest) request;
     }
-    return new MysqlFetcherRequest(request);
-  }
-
-  /**
-   * Get content type from record, if not found return default.
-   *
-   * @param record an instance of record
-   * @return an instance of content type
-   */
-  private ContentType getContentType(final Record record) {
-    try {
-      return ContentType.create(record.getMimeType(), record.getEncoding());
-    } catch (ParseException e) {
-      LOGGER.warn("Could not parse content type", e);
-    } catch (UnsupportedCharsetException e) {
-      LOGGER.warn("Charset is not available in this instance of the Java virtual machine", e);
-    }
-    return DEFAULT_CONTENT_TYPE;
+    return new StorageFetcherRequest(request);
   }
 
   @Override
@@ -143,7 +116,7 @@ public Future<Response> fetch(final Request request) {
   @Override
   public Future<Response> fetch(final Request request, final Callback callback) {
     LOGGER.debug("Getting record for: {}", request.getUrl());
-    final MysqlFetcherRequest mysqlFetcherRequest = normalize(request).prependHeaders(headers);
+    final StorageFetcherRequest storageFetcherRequest = normalize(request).prependHeaders(headers);
 
     final BasicFuture<Response> future = new BasicFuture<>(new FutureCallback<Response>() {
       @Override
@@ -163,46 +136,35 @@ public void cancelled() {
     });
 
     try {
-      final Record record = fileManager.get(mysqlFetcherRequest);
+      final Record record = fileManager.get(storageFetcherRequest);
+      if (record == null) {
+        future.cancel();
+        LOGGER.info("No content found from storage for: {}", request.getUrl());
+        return future;
+      }
+
       LOGGER.debug("Record found with id: {}", record.getId());
 
-      String baseUrl;
+      String tryBaseUrl;
       try {
-        baseUrl = UrlUtil.getBaseUrl(request);
+        tryBaseUrl = UrlUtil.getBaseUrl(request);
       } catch (URISyntaxException e) {
         LOGGER.warn("Could not parse base URL: " + request.getUrl());
-        baseUrl = request.getUrl();
+        tryBaseUrl = request.getUrl();
       }
-
-      final StorageResponse response = new StorageResponse(
-          record.getStatusCode(),
-          baseUrl,
-          record.getResponseContent(),
-          getContentType(record),
-          record.getResponseHeaders(),
-          String.valueOf(record.getId())
-      );
-
+      final String baseUrl = tryBaseUrl;
+      final StorageResponse response = new StorageResponse(record, baseUrl);
       final Validator.Status status = validator.isValid(Unwrappable.unwrapRequest(request), response);
       if (status != Validator.Status.VALID) {
         future.failed(new ValidationException(status, response, "Invalid response."));
         return future;
       }
 
-      final Response validatedResponse = new StorageResponse(response, validator);
-      future.completed(validatedResponse);
-      return future;
-    } catch (MalformedURLException e) {
-      LOGGER.warn("Could not parse base URL: " + request.getUrl());
-      future.failed(e);
-      return future;
-    } catch (IOException e) {
-      LOGGER.error("Fail to parse content from storage", e);
-      future.failed(e);
+      future.completed(response);
       return future;
     } catch (StorageException e) {
-      LOGGER.warn("No content found from storage for: {}", request.getUrl());
-      future.cancel();
+      LOGGER.warn("Error retrieving content for : {}", request.getUrl(), e);
+      future.failed(e);
       return future;
     }
   }
@@ -215,7 +177,7 @@ public void close() throws Exception {
   }
 
   /**
-   * A builder for MySQL fetcher class.
+   * A builder for StorageFetcher class.
    */
   public static final class Builder {
 
@@ -296,8 +258,8 @@ public Builder setValidator(final @NotNull Validator... validators) {
      *
      * @return an instance of Fetcher.
      */
-    public MysqlFetcher build() {
-      return new MysqlFetcher(this);
+    public StorageFetcher build() {
+      return new StorageFetcher(this);
     }
 
   }

src/main/java/ai/preferred/venom/request/MysqlFetcherRequest.java → src/main/java/ai/preferred/venom/request/StorageFetcherRequest.java

@@ -27,7 +27,7 @@
 /**
  * @author Ween Jiann Lee
  */
-public class MysqlFetcherRequest implements Request, Unwrappable {
+public class StorageFetcherRequest implements Request, Unwrappable {
 
   /**
    * An instance of underlying request.
@@ -40,21 +40,21 @@ public class MysqlFetcherRequest implements Request, Unwrappable {
   private final Map<String, String> headers;
 
   /**
-   * Constructs an instance of mysql fetcher request.
+   * Constructs an instance of StorageFetcherRequest.
    *
    * @param innerRequest An instance of underlying request
    */
-  public MysqlFetcherRequest(final Request innerRequest) {
+  public StorageFetcherRequest(final Request innerRequest) {
     this(innerRequest, new HashMap<>(innerRequest.getHeaders()));
   }
 
   /**
-   * Constructs an instance of mysql fetcher request.
+   * Constructs an instance of StorageFetcherRequest.
    *
    * @param innerRequest An instance of underlying request
    * @param headers      Headers to append to global headers
    */
-  private MysqlFetcherRequest(final Request innerRequest, final Map<String, String> headers) {
+  private StorageFetcherRequest(final Request innerRequest, final Map<String, String> headers) {
     this.innerRequest = innerRequest;
     this.headers = headers;
   }
@@ -65,10 +65,10 @@ private MysqlFetcherRequest(final Request innerRequest, final Map<String, String
    * @param preHeaders Headers to be prepended
    * @return A new instance of http fetcher request
    */
-  public final MysqlFetcherRequest prependHeaders(final Map<String, String> preHeaders) {
+  public final StorageFetcherRequest prependHeaders(final Map<String, String> preHeaders) {
     final Map<String, String> newHeaders = new HashMap<>(headers);
     preHeaders.forEach(newHeaders::putIfAbsent);
-    return new MysqlFetcherRequest(innerRequest, newHeaders);
+    return new StorageFetcherRequest(innerRequest, newHeaders);
   }
 
   @Override

src/main/java/ai/preferred/venom/response/BaseResponse.java

@@ -16,7 +16,6 @@
 
 package ai.preferred.venom.response;
 
-import ai.preferred.venom.validator.Validator;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
@@ -26,7 +25,7 @@
  * @author Truong Quoc Tuan
  * @author Ween Jiann Lee
  */
-public class BaseResponse implements Response, Retrievable {
+public class BaseResponse implements Response {
 
   /**
    * The status code of this response.
@@ -58,16 +57,6 @@ public class BaseResponse implements Response, Retrievable {
    */
   private final HttpHost proxy;
 
-  /**
-   * The validator used to validate this response.
-   */
-  private final Validator validator;
-
-  /**
-   * The source id of this response.
-   */
-  private final String sourceId;
-
   /**
    * Constructs a base response.
    *
@@ -80,31 +69,12 @@ public class BaseResponse implements Response, Retrievable {
    */
   public BaseResponse(final int statusCode, final String baseUrl, final byte[] content, final ContentType contentType,
                       final Header[] headers, final HttpHost proxy) {
-    this(statusCode, baseUrl, content, contentType, headers, proxy, null, null);
-  }
-
-  /**
-   * Constructs a base response.
-   *
-   * @param statusCode  Status code of the response
-   * @param baseUrl     Base url of the response
-   * @param content     Content from the response
-   * @param contentType Content type of the response
-   * @param headers     Headers from the response
-   * @param proxy       Proxy used to obtain the response
-   * @param validator   Validator used to validate this response
-   * @param sourceId    `id` of the row the raw response is saved to
-   */
-  public BaseResponse(final int statusCode, final String baseUrl, final byte[] content, final ContentType contentType,
-                      final Header[] headers, final HttpHost proxy, final Validator validator, final String sourceId) {
     this.statusCode = statusCode;
     this.baseUrl = baseUrl;
     this.content = content;
     this.contentType = contentType;
     this.headers = headers;
     this.proxy = proxy;
-    this.validator = validator;
-    this.sourceId = sourceId;
   }
 
   @Override
@@ -137,34 +107,4 @@ public final HttpHost getProxy() {
     return proxy;
   }
 
-  @Override
-  public final Validator getValidator() {
-    return validator;
-  }
-
-  /**
-   * Sets the validator used to validate this response.
-   *
-   * @param validator Row id of the saved response
-   * @return A new instance of base response
-   */
-  public final BaseResponse setValidator(final Validator validator) {
-    return new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy, validator, sourceId);
-  }
-
-  @Override
-  public final String getSourceId() {
-    return sourceId;
-  }
-
-  /**
-   * Sets the source id where the raw response is saved.
-   *
-   * @param sourceId Row id of the saved response
-   * @return A new instance of base response
-   */
-  public final BaseResponse setSourceId(final String sourceId) {
-    return new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy, validator, sourceId);
-  }
-
 }

src/main/java/ai/preferred/venom/response/Response.java

@@ -16,7 +16,6 @@
 
 package ai.preferred.venom.response;
 
-import ai.preferred.venom.validator.Validator;
 import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.entity.ContentType;
@@ -81,12 +80,4 @@ public interface Response {
   @Nullable
   HttpHost getProxy();
 
-  /**
-   * Returns the instance of validator used to validate this response.
-   *
-   * @return an instance of validator
-   */
-  @NotNull
-  Validator getValidator();
-
 }