diff --git a/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java b/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java index 0aef591..89c02df 100644 --- a/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java +++ b/src/main/java/ai/preferred/venom/ThreadedWorkerManager.java @@ -49,7 +49,7 @@ public class ThreadedWorkerManager implements WorkerManager { * * @param executor An executor service */ - public ThreadedWorkerManager(final ExecutorService executor) { + public ThreadedWorkerManager(@Nullable final ExecutorService executor) { this.executor = executor; if (executor instanceof ForkJoinPool || executor == null) { this.worker = new ForkJoinWorker(); diff --git a/src/main/java/ai/preferred/venom/fetcher/Callback.java b/src/main/java/ai/preferred/venom/fetcher/Callback.java index 8d394c5..b3e39d8 100644 --- a/src/main/java/ai/preferred/venom/fetcher/Callback.java +++ b/src/main/java/ai/preferred/venom/fetcher/Callback.java @@ -32,17 +32,17 @@ public interface Callback { Callback EMPTY_CALLBACK = new Callback() { @Override public void completed(final @NotNull Request request, final @NotNull Response response) { - // Do nothing + // do nothing } @Override public void failed(final @NotNull Request request, final @NotNull Exception ex) { - // Do nothing + // do nothing } @Override public void cancelled(final @NotNull Request request) { - // Do nothing + // do nothing } }; diff --git a/src/main/java/ai/preferred/venom/fetcher/MysqlFetcher.java b/src/main/java/ai/preferred/venom/fetcher/StorageFetcher.java similarity index 68% rename from src/main/java/ai/preferred/venom/fetcher/MysqlFetcher.java rename to src/main/java/ai/preferred/venom/fetcher/StorageFetcher.java index 0683176..b83bf7f 100644 --- a/src/main/java/ai/preferred/venom/fetcher/MysqlFetcher.java +++ b/src/main/java/ai/preferred/venom/fetcher/StorageFetcher.java @@ -16,8 +16,8 @@ package ai.preferred.venom.fetcher; -import ai.preferred.venom.request.MysqlFetcherRequest; import ai.preferred.venom.request.Request; +import ai.preferred.venom.request.StorageFetcherRequest; import ai.preferred.venom.request.Unwrappable; import ai.preferred.venom.response.Response; import ai.preferred.venom.response.StorageResponse; @@ -29,39 +29,29 @@ import ai.preferred.venom.validator.PipelineValidator; import ai.preferred.venom.validator.StatusOkValidator; import ai.preferred.venom.validator.Validator; -import org.apache.http.ParseException; import org.apache.http.concurrent.BasicFuture; import org.apache.http.concurrent.FutureCallback; -import org.apache.http.entity.ContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.validation.constraints.NotNull; -import java.io.IOException; -import java.net.MalformedURLException; import java.net.URISyntaxException; -import java.nio.charset.UnsupportedCharsetException; import java.util.Collections; import java.util.Map; import java.util.concurrent.Future; /** - * This class holds the implementation to provide how items are fetched from a MySQL database, + * This class holds the implementation to provide how items are fetched from a database, * to validate the item and to store it if specified. * * @author Ween Jiann Lee */ -public final class MysqlFetcher implements Fetcher { +public final class StorageFetcher implements Fetcher { /** * Logger. */ - private static final Logger LOGGER = LoggerFactory.getLogger(MysqlFetcher.class); - - /** - * Default content type of response if not given. - */ - private static final ContentType DEFAULT_CONTENT_TYPE = ContentType.APPLICATION_OCTET_STREAM; + private static final Logger LOGGER = LoggerFactory.getLogger(StorageFetcher.class); /** * The file manager used to store raw responses. @@ -79,11 +69,11 @@ public final class MysqlFetcher implements Fetcher { private final Map headers; /** - * Constructs an instance of MySQL fetcher. + * Constructs an instance of StorageFetcher. * * @param builder An instance of builder */ - private MysqlFetcher(final Builder builder) { + private StorageFetcher(final Builder builder) { this.fileManager = builder.fileManager; this.validator = builder.validator; this.headers = builder.headers; @@ -100,34 +90,17 @@ public static Builder builder(final FileManager fileManager) { } /** - * Check if request is an instance of MySQL fetcher request and return it - * if true, otherwise wrap it with MySQL fetcher request and return that. + * Check if request is an instance of StorageFetcher request and return it + * if true, otherwise wrap it with StorageFetcherRequest and return that. * * @param request An instance of request - * @return An instance of MySQL fetcher request + * @return An instance of StorageFetcherRequest */ - private MysqlFetcherRequest normalize(final Request request) { - if (request instanceof MysqlFetcherRequest) { - return (MysqlFetcherRequest) request; + private StorageFetcherRequest normalize(final Request request) { + if (request instanceof StorageFetcherRequest) { + return (StorageFetcherRequest) request; } - return new MysqlFetcherRequest(request); - } - - /** - * Get content type from record, if not found return default. - * - * @param record an instance of record - * @return an instance of content type - */ - private ContentType getContentType(final Record record) { - try { - return ContentType.create(record.getMimeType(), record.getEncoding()); - } catch (ParseException e) { - LOGGER.warn("Could not parse content type", e); - } catch (UnsupportedCharsetException e) { - LOGGER.warn("Charset is not available in this instance of the Java virtual machine", e); - } - return DEFAULT_CONTENT_TYPE; + return new StorageFetcherRequest(request); } @Override @@ -143,7 +116,7 @@ public Future fetch(final Request request) { @Override public Future fetch(final Request request, final Callback callback) { LOGGER.debug("Getting record for: {}", request.getUrl()); - final MysqlFetcherRequest mysqlFetcherRequest = normalize(request).prependHeaders(headers); + final StorageFetcherRequest storageFetcherRequest = normalize(request).prependHeaders(headers); final BasicFuture future = new BasicFuture<>(new FutureCallback() { @Override @@ -163,46 +136,35 @@ public void cancelled() { }); try { - final Record record = fileManager.get(mysqlFetcherRequest); + final Record record = fileManager.get(storageFetcherRequest); + if (record == null) { + future.cancel(); + LOGGER.info("No content found from storage for: {}", request.getUrl()); + return future; + } + LOGGER.debug("Record found with id: {}", record.getId()); - String baseUrl; + String tryBaseUrl; try { - baseUrl = UrlUtil.getBaseUrl(request); + tryBaseUrl = UrlUtil.getBaseUrl(request); } catch (URISyntaxException e) { LOGGER.warn("Could not parse base URL: " + request.getUrl()); - baseUrl = request.getUrl(); + tryBaseUrl = request.getUrl(); } - - final StorageResponse response = new StorageResponse( - record.getStatusCode(), - baseUrl, - record.getResponseContent(), - getContentType(record), - record.getResponseHeaders(), - String.valueOf(record.getId()) - ); - + final String baseUrl = tryBaseUrl; + final StorageResponse response = new StorageResponse(record, baseUrl); final Validator.Status status = validator.isValid(Unwrappable.unwrapRequest(request), response); if (status != Validator.Status.VALID) { future.failed(new ValidationException(status, response, "Invalid response.")); return future; } - final Response validatedResponse = new StorageResponse(response, validator); - future.completed(validatedResponse); - return future; - } catch (MalformedURLException e) { - LOGGER.warn("Could not parse base URL: " + request.getUrl()); - future.failed(e); - return future; - } catch (IOException e) { - LOGGER.error("Fail to parse content from storage", e); - future.failed(e); + future.completed(response); return future; } catch (StorageException e) { - LOGGER.warn("No content found from storage for: {}", request.getUrl()); - future.cancel(); + LOGGER.warn("Error retrieving content for : {}", request.getUrl(), e); + future.failed(e); return future; } } @@ -215,7 +177,7 @@ public void close() throws Exception { } /** - * A builder for MySQL fetcher class. + * A builder for StorageFetcher class. */ public static final class Builder { @@ -296,8 +258,8 @@ public Builder setValidator(final @NotNull Validator... validators) { * * @return an instance of Fetcher. */ - public MysqlFetcher build() { - return new MysqlFetcher(this); + public StorageFetcher build() { + return new StorageFetcher(this); } } diff --git a/src/main/java/ai/preferred/venom/request/MysqlFetcherRequest.java b/src/main/java/ai/preferred/venom/request/StorageFetcherRequest.java similarity index 83% rename from src/main/java/ai/preferred/venom/request/MysqlFetcherRequest.java rename to src/main/java/ai/preferred/venom/request/StorageFetcherRequest.java index afca14a..3cc3300 100644 --- a/src/main/java/ai/preferred/venom/request/MysqlFetcherRequest.java +++ b/src/main/java/ai/preferred/venom/request/StorageFetcherRequest.java @@ -27,7 +27,7 @@ /** * @author Ween Jiann Lee */ -public class MysqlFetcherRequest implements Request, Unwrappable { +public class StorageFetcherRequest implements Request, Unwrappable { /** * An instance of underlying request. @@ -40,21 +40,21 @@ public class MysqlFetcherRequest implements Request, Unwrappable { private final Map headers; /** - * Constructs an instance of mysql fetcher request. + * Constructs an instance of StorageFetcherRequest. * * @param innerRequest An instance of underlying request */ - public MysqlFetcherRequest(final Request innerRequest) { + public StorageFetcherRequest(final Request innerRequest) { this(innerRequest, new HashMap<>(innerRequest.getHeaders())); } /** - * Constructs an instance of mysql fetcher request. + * Constructs an instance of StorageFetcherRequest. * * @param innerRequest An instance of underlying request * @param headers Headers to append to global headers */ - private MysqlFetcherRequest(final Request innerRequest, final Map headers) { + private StorageFetcherRequest(final Request innerRequest, final Map headers) { this.innerRequest = innerRequest; this.headers = headers; } @@ -65,10 +65,10 @@ private MysqlFetcherRequest(final Request innerRequest, final Map preHeaders) { + public final StorageFetcherRequest prependHeaders(final Map preHeaders) { final Map newHeaders = new HashMap<>(headers); preHeaders.forEach(newHeaders::putIfAbsent); - return new MysqlFetcherRequest(innerRequest, newHeaders); + return new StorageFetcherRequest(innerRequest, newHeaders); } @Override diff --git a/src/main/java/ai/preferred/venom/response/BaseResponse.java b/src/main/java/ai/preferred/venom/response/BaseResponse.java index 8b5ba43..a04e2f1 100644 --- a/src/main/java/ai/preferred/venom/response/BaseResponse.java +++ b/src/main/java/ai/preferred/venom/response/BaseResponse.java @@ -16,7 +16,6 @@ package ai.preferred.venom.response; -import ai.preferred.venom.validator.Validator; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; @@ -26,7 +25,7 @@ * @author Truong Quoc Tuan * @author Ween Jiann Lee */ -public class BaseResponse implements Response, Retrievable { +public class BaseResponse implements Response { /** * The status code of this response. @@ -58,16 +57,6 @@ public class BaseResponse implements Response, Retrievable { */ private final HttpHost proxy; - /** - * The validator used to validate this response. - */ - private final Validator validator; - - /** - * The source id of this response. - */ - private final String sourceId; - /** * Constructs a base response. * @@ -80,31 +69,12 @@ public class BaseResponse implements Response, Retrievable { */ public BaseResponse(final int statusCode, final String baseUrl, final byte[] content, final ContentType contentType, final Header[] headers, final HttpHost proxy) { - this(statusCode, baseUrl, content, contentType, headers, proxy, null, null); - } - - /** - * Constructs a base response. - * - * @param statusCode Status code of the response - * @param baseUrl Base url of the response - * @param content Content from the response - * @param contentType Content type of the response - * @param headers Headers from the response - * @param proxy Proxy used to obtain the response - * @param validator Validator used to validate this response - * @param sourceId `id` of the row the raw response is saved to - */ - public BaseResponse(final int statusCode, final String baseUrl, final byte[] content, final ContentType contentType, - final Header[] headers, final HttpHost proxy, final Validator validator, final String sourceId) { this.statusCode = statusCode; this.baseUrl = baseUrl; this.content = content; this.contentType = contentType; this.headers = headers; this.proxy = proxy; - this.validator = validator; - this.sourceId = sourceId; } @Override @@ -137,34 +107,4 @@ public final HttpHost getProxy() { return proxy; } - @Override - public final Validator getValidator() { - return validator; - } - - /** - * Sets the validator used to validate this response. - * - * @param validator Row id of the saved response - * @return A new instance of base response - */ - public final BaseResponse setValidator(final Validator validator) { - return new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy, validator, sourceId); - } - - @Override - public final String getSourceId() { - return sourceId; - } - - /** - * Sets the source id where the raw response is saved. - * - * @param sourceId Row id of the saved response - * @return A new instance of base response - */ - public final BaseResponse setSourceId(final String sourceId) { - return new BaseResponse(statusCode, baseUrl, content, contentType, headers, proxy, validator, sourceId); - } - } diff --git a/src/main/java/ai/preferred/venom/response/Response.java b/src/main/java/ai/preferred/venom/response/Response.java index 0e09bd5..09e0020 100644 --- a/src/main/java/ai/preferred/venom/response/Response.java +++ b/src/main/java/ai/preferred/venom/response/Response.java @@ -16,7 +16,6 @@ package ai.preferred.venom.response; -import ai.preferred.venom.validator.Validator; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; @@ -81,12 +80,4 @@ public interface Response { @Nullable HttpHost getProxy(); - /** - * Returns the instance of validator used to validate this response. - * - * @return an instance of validator - */ - @NotNull - Validator getValidator(); - } diff --git a/src/main/java/ai/preferred/venom/response/Retrievable.java b/src/main/java/ai/preferred/venom/response/Retrievable.java index ffa7684..0e140d2 100644 --- a/src/main/java/ai/preferred/venom/response/Retrievable.java +++ b/src/main/java/ai/preferred/venom/response/Retrievable.java @@ -16,6 +16,8 @@ package ai.preferred.venom.response; +import ai.preferred.venom.storage.Record; + /** * This interface represents that the response can be/ has been stored. * @@ -24,14 +26,10 @@ public interface Retrievable extends Response { /** - * Returns the id of the row where an archive of this response - * has been insert into a persistent storage. - *

- * This string should be null if no implementation of FileManager - * is specified during the initialisation of the fetcher. - *

+ * Returns record archive of this response that has been + * insert into a persistent storage. * - * @return unique id of where an archive has been saved + * @return record where an archive has been saved */ - String getSourceId(); + Record getRecord(); } diff --git a/src/main/java/ai/preferred/venom/response/StorageResponse.java b/src/main/java/ai/preferred/venom/response/StorageResponse.java index 9e4bf51..1efb3df 100644 --- a/src/main/java/ai/preferred/venom/response/StorageResponse.java +++ b/src/main/java/ai/preferred/venom/response/StorageResponse.java @@ -16,7 +16,7 @@ package ai.preferred.venom.response; -import ai.preferred.venom.validator.Validator; +import ai.preferred.venom.storage.Record; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; @@ -28,107 +28,44 @@ public class StorageResponse implements Response, Retrievable { /** - * The status code of this response. + * The record holding this response. */ - private final int statusCode; - - /** - * The content of this response. - */ - private final byte[] content; - - /** - * The content type of this response. - */ - private final ContentType contentType; - - /** - * The headers of this response. - */ - private final Header[] headers; + private final Record record; /** * The base url of this response. */ private final String baseUrl; - /** - * The validator used to validate this response. - */ - private final Validator validator; - - /** - * The source id of this response. - */ - private final String sourceId; - /** * Constructs a base response. * - * @param statusCode Status code of the response - * @param baseUrl Status code of the response - * @param content Content from the response - * @param contentType Content type of the response - * @param headers Headers from the response - * @param sourceId Stored ID of the response + * @param record record holding this response + * @param baseUrl base URL of the response */ - public StorageResponse(final int statusCode, final String baseUrl, final byte[] content, - final ContentType contentType, final Header[] headers, final String sourceId) { - this(statusCode, baseUrl, content, contentType, headers, sourceId, null); - } - - /** - * Constructs a base response. - * - * @param response an instance of storage response where validator is to be replaced - * @param validator Validator used to validate this response - */ - public StorageResponse(final StorageResponse response, final Validator validator) { - this(response.getStatusCode(), response.getBaseUrl(), response.getContent(), response.getContentType(), - response.getHeaders(), response.getSourceId(), validator); - } - - /** - * Constructs a base response. - * - * @param statusCode Status code of the response - * @param baseUrl Status code of the response - * @param content Content from the response - * @param contentType Content type of the response - * @param headers Headers from the response - * @param sourceId Stored ID of the response - * @param validator Validator used to validate this response - */ - private StorageResponse(final int statusCode, final String baseUrl, final byte[] content, - final ContentType contentType, final Header[] headers, final String sourceId, - final Validator validator) { - this.statusCode = statusCode; + public StorageResponse(final Record record, final String baseUrl) { + this.record = record; this.baseUrl = baseUrl; - this.content = content; - this.contentType = contentType; - this.headers = headers; - this.sourceId = sourceId; - this.validator = validator; } @Override public final int getStatusCode() { - return statusCode; + return record.getStatusCode(); } @Override public final byte[] getContent() { - return content; + return record.getResponseContent(); } @Override public final ContentType getContentType() { - return contentType; + return record.getContentType(); } @Override public final Header[] getHeaders() { - return headers; + return record.getResponseHeaders(); } @Override @@ -136,18 +73,13 @@ public final String getBaseUrl() { return baseUrl; } - @Override - public final Validator getValidator() { - return validator; - } - @Override public final HttpHost getProxy() { return null; } @Override - public final String getSourceId() { - return sourceId; + public final Record getRecord() { + return record; } } diff --git a/src/main/java/ai/preferred/venom/response/VResponse.java b/src/main/java/ai/preferred/venom/response/VResponse.java index 8c08fbf..21c539b 100644 --- a/src/main/java/ai/preferred/venom/response/VResponse.java +++ b/src/main/java/ai/preferred/venom/response/VResponse.java @@ -16,7 +16,6 @@ package ai.preferred.venom.response; -import ai.preferred.venom.validator.Validator; import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; @@ -82,11 +81,6 @@ public final HttpHost getProxy() { return getInner().getProxy(); } - @Override - public final Validator getValidator() { - return getInner().getValidator(); - } - /** * Returns the html in string format. * diff --git a/src/main/java/ai/preferred/venom/storage/DummyFileManager.java b/src/main/java/ai/preferred/venom/storage/DummyFileManager.java index 1700698..42cbb50 100644 --- a/src/main/java/ai/preferred/venom/storage/DummyFileManager.java +++ b/src/main/java/ai/preferred/venom/storage/DummyFileManager.java @@ -46,9 +46,9 @@ public class DummyFileManager implements FileManager { private static final Logger LOGGER = LoggerFactory.getLogger(DummyFileManager.class); /** - * The directory path to store raw responses. + * The storage path on the file system to use for content storage. */ - private final String storageDir; + private final File storagePath; /** * The callback to trigger upon response. @@ -58,10 +58,19 @@ public class DummyFileManager implements FileManager { /** * Constructs an instance of DummyFileManager. * - * @param storageDir directory path to store raw responses + * @param storageDir storage directory to use for content storage */ public DummyFileManager(final String storageDir) { - this.storageDir = storageDir; + this(new File(storageDir)); + } + + /** + * Constructs an instance of DummyFileManager. + * + * @param storagePath storage path to use for content storage + */ + public DummyFileManager(final File storagePath) { + this.storagePath = storagePath; this.callback = new FileManagerCallback(this); } @@ -113,14 +122,14 @@ public final String put(final Request request, final Response response) throws S final String fileExtension = tryFileExtension; LOGGER.info("Response from request {} has been saved to {}", request.getUrl(), md5 + fileExtension); - return write(content, new File(storageDir, subDirName), md5 + fileExtension); + return write(content, new File(storagePath, subDirName), md5 + fileExtension); } catch (IOException e) { throw new StorageException("Error in put.", e); } } @Override - public final Record get(final int i) { + public final Record get(final Object i) { throw new UnsupportedOperationException("File not found"); } diff --git a/src/main/java/ai/preferred/venom/storage/FileManager.java b/src/main/java/ai/preferred/venom/storage/FileManager.java index 3187370..15d641e 100644 --- a/src/main/java/ai/preferred/venom/storage/FileManager.java +++ b/src/main/java/ai/preferred/venom/storage/FileManager.java @@ -20,13 +20,14 @@ import ai.preferred.venom.request.Request; import ai.preferred.venom.response.Response; +import javax.annotation.Nullable; import javax.validation.constraints.NotNull; /** * @author Maksim Tkachenko * @author Truong Quoc Tuan */ -public interface FileManager extends AutoCloseable { +public interface FileManager extends AutoCloseable { /** * Get callback upon completion of request. @@ -59,8 +60,8 @@ public interface FileManager extends AutoCloseable { * @return stored record * @throws StorageException throws StorageException */ - @NotNull - Record get(int id) throws StorageException; + @Nullable + Record get(T id) throws StorageException; /** * Returns latest record matching request. diff --git a/src/main/java/ai/preferred/venom/storage/FileManagerCallback.java b/src/main/java/ai/preferred/venom/storage/FileManagerCallback.java index a197e46..2b97d50 100644 --- a/src/main/java/ai/preferred/venom/storage/FileManagerCallback.java +++ b/src/main/java/ai/preferred/venom/storage/FileManagerCallback.java @@ -59,12 +59,12 @@ public final void completed(final Request request, final Response response) { @Override public final void failed(final Request request, final Exception ex) { - // Do nothing + // do nothing } @Override public final void cancelled(final Request request) { - // Do nothing + // do nothing } } diff --git a/src/main/java/ai/preferred/venom/storage/MysqlFileManager.java b/src/main/java/ai/preferred/venom/storage/MysqlFileManager.java index fed1eaa..6e2b7e6 100644 --- a/src/main/java/ai/preferred/venom/storage/MysqlFileManager.java +++ b/src/main/java/ai/preferred/venom/storage/MysqlFileManager.java @@ -24,6 +24,8 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; import org.apache.http.Header; +import org.apache.http.ParseException; +import org.apache.http.entity.ContentType; import org.apache.http.message.BasicHeader; import org.apache.tika.mime.MimeTypeException; import org.json.JSONObject; @@ -34,10 +36,12 @@ import javax.validation.constraints.NotNull; import java.io.*; import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; import java.sql.*; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; /** @@ -49,13 +53,18 @@ * @author Truong Quoc Tuan * @author Ween Jiann Lee */ -public class MysqlFileManager implements FileManager { +public class MysqlFileManager implements FileManager { /** * Logger. */ private static final Logger LOGGER = LoggerFactory.getLogger(MysqlFileManager.class); + /** + * Default content type of response if not given. + */ + private static final ContentType DEFAULT_CONTENT_TYPE = ContentType.APPLICATION_OCTET_STREAM; + /** * The DataSource to use for connecting to database. */ @@ -232,6 +241,30 @@ private Map prepareRequestBody(final Request request) { return requestBody; } + /** + * Get content type from record, if not found return default. + * + * @param mimeType name of mime type + * @param encoding name of encoding + * @return an instance of content type + */ + private ContentType getContentType(final String mimeType, final String encoding) { + final Charset charset; + if (encoding != null) { + charset = Charset.forName(encoding); + } else { + charset = null; + } + try { + return ContentType.create(mimeType, charset); + } catch (ParseException e) { + LOGGER.warn("Could not parse content type", e); + } catch (UnsupportedCharsetException e) { + LOGGER.warn("Charset is not available in this instance of the Java virtual machine", e); + } + return DEFAULT_CONTENT_TYPE; + } + /** * Create an instance of Record using the result from database. * @@ -242,7 +275,7 @@ private Map prepareRequestBody(final Request request) { * called on a closed result set * @throws StorageException if file is not found */ - private StorageRecord createRecord(final ResultSet rs) throws SQLException, StorageException { + private StorageRecord createRecord(final ResultSet rs) throws SQLException, StorageException { final Map requestHeaders = parseRequestHeaders(new JSONObject(rs.getString("request_headers"))); final Header[] responseHeaders = parseResponseHeaders(new JSONObject(rs.getString("response_headers"))); final String location = rs.getString("location"); @@ -256,28 +289,37 @@ private StorageRecord createRecord(final ResultSet rs) throws SQLException, Stor final String fileExtension = tryFileExtension; final File file = new File(new File(storagePath, location), rs.getString("id") + fileExtension + ".gz"); - final StorageRecord.Builder builder = StorageRecord.builder() + final ContentType contentType = getContentType( + rs.getString("mime_type"), rs.getString("encoding")); + + final byte[] responseContent; + try { + responseContent = IOUtils.toByteArray( + new BufferedInputStream( + new GZIPInputStream( + new FileInputStream(file) + ) + ) + ); + } catch (FileNotFoundException e) { + throw new StorageException("Record found but file not found for " + rs.getString("url") + ".", e); + } catch (IOException e) { + throw new StorageException("Error reading file for " + rs.getString("url") + ".", e); + } + + LOGGER.debug("Record found for request: {}", rs.getString("url")); + //noinspection unchecked + return StorageRecord.builder() .setId(rs.getInt("id")) .setUrl(rs.getString("url")) .setRequestMethod(Request.Method.valueOf(rs.getString("method"))) .setRequestHeaders(requestHeaders) .setResponseHeaders(responseHeaders) - .setMimeType(rs.getString("mime_type")) + .setContentType(contentType) .setMD5(rs.getString("md5")) .setDateCreated(rs.getLong("date_created")) - .setResponseContent(file); - - if (rs.getString("encoding") != null) { - builder.setEncoding(Charset.forName(rs.getString("encoding"))); - } - - if (file.exists()) { - LOGGER.debug("Record found for request: {}", rs.getString("url")); - return builder.build(); - } - - LOGGER.error("Record found but file not found for request: {}", rs.getString("url")); - throw new StorageException("Record found but file not found"); + .setResponseContent(responseContent) + .build(); } @Override @@ -373,7 +415,7 @@ public final String put(final Request request, final Response response) throws S } @Override - public final Record get(final int id) throws StorageException { + public final Record get(final Integer id) throws StorageException { try (final Connection conn = dataSource.getConnection(); final PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM `" + table + "` WHERE id = ?")) { pstmt.setInt(1, id); @@ -381,16 +423,17 @@ public final Record get(final int id) throws StorageException { if (rs.next()) { return createRecord(rs); } + } catch (SQLException e) { LOGGER.error("Record query failure for id: {}", id, e); throw new StorageException("Cannot retrieve the record", e); } LOGGER.debug("No record found for id: {}", id); - throw new StorageException("No record found"); + return null; } @Override - public final Record get(final Request request) throws StorageException { + public final Record get(final Request request) throws StorageException { try (final Connection conn = dataSource.getConnection(); final PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM `" + table + "` " + "WHERE url = ? " @@ -407,12 +450,13 @@ public final Record get(final Request request) throws StorageException { if (rs.next()) { return createRecord(rs); } + } catch (SQLException e) { LOGGER.error("Record query failure for request: {}", request.getUrl(), e); - throw new StorageException("Cannot retrieve the record", e); + throw new StorageException("Cannot retrieve the record for " + request.getUrl() + ".", e); } LOGGER.debug("No record found for request: {}", request.getUrl()); - throw new StorageException("No record found"); + return null; } @Override diff --git a/src/main/java/ai/preferred/venom/storage/Record.java b/src/main/java/ai/preferred/venom/storage/Record.java index 0e0bdc6..058b9d1 100644 --- a/src/main/java/ai/preferred/venom/storage/Record.java +++ b/src/main/java/ai/preferred/venom/storage/Record.java @@ -18,25 +18,27 @@ import ai.preferred.venom.request.Request; import org.apache.http.Header; +import org.apache.http.entity.ContentType; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; import java.util.Map; /** + * This interface represents only the most basic of a record and + * the fields that should be retrievable from database. + * + * @param the type of id * @author Maksim Tkachenko * @author Truong Quoc Tuan * @author Ween Jiann Lee */ -public interface Record { +public interface Record { /** * @return valid id if the record is stored, null otherwise */ - int getId(); + T getId(); /** * @return URL of the stored content @@ -55,14 +57,9 @@ public interface Record { Map getRequestHeaders(); /** - * @return mime type of the content - */ - String getMimeType(); - - /** - * @return encoding of the content + * @return ContentType of the content */ - Charset getEncoding(); + ContentType getContentType(); /** * @return map of request body @@ -83,16 +80,8 @@ public interface Record { /** * @return raw response file (uncompressed) - * @throws IOException throws IOException - */ - byte[] getResponseContent() throws IOException; - - /** - * @return response file (uncompressed) - * @throws IOException throws IOException */ - @NotNull - InputStream getStreamResponseContent() throws IOException; + byte[] getResponseContent(); /** * @return valid timestamp if the record is stored, -1 otherwise diff --git a/src/main/java/ai/preferred/venom/storage/StorageRecord.java b/src/main/java/ai/preferred/venom/storage/StorageRecord.java index b71027c..4314ba6 100644 --- a/src/main/java/ai/preferred/venom/storage/StorageRecord.java +++ b/src/main/java/ai/preferred/venom/storage/StorageRecord.java @@ -17,25 +17,23 @@ package ai.preferred.venom.storage; import ai.preferred.venom.request.Request; -import org.apache.commons.io.IOUtils; import org.apache.http.Header; +import org.apache.http.entity.ContentType; -import java.io.*; -import java.nio.charset.Charset; import java.util.Map; -import java.util.zip.GZIPInputStream; /** * This class implements a default storage record. * + * @param the type of id * @author Ween Jiann Lee */ -public final class StorageRecord implements Record { +public final class StorageRecord implements Record { /** * The id of this record. */ - private final int id; + private final T id; /** * The url for this request. @@ -68,19 +66,14 @@ public final class StorageRecord implements Record { private final Header[] responseHeaders; /** - * The mime type of the response. + * The content type of the response. */ - private final String mimeType; - - /** - * The encoding of the response. - */ - private final Charset encoding; + private final ContentType contentType; /** * The content of the response. */ - private final File responseContent; + private final byte[] responseContent; /** * The md5 hash of the content. @@ -97,7 +90,7 @@ public final class StorageRecord implements Record { * * @param builder an instance of builder */ - private StorageRecord(final Builder builder) { + private StorageRecord(final Builder builder) { this.id = builder.id; this.url = builder.url; this.requestMethod = builder.requestMethod; @@ -105,8 +98,7 @@ private StorageRecord(final Builder builder) { this.requestBody = builder.requestBody; this.statusCode = builder.statusCode; this.responseHeaders = builder.responseHeaders; - this.mimeType = builder.mimeType; - this.encoding = builder.encoding; + this.contentType = builder.contentType; this.responseContent = builder.responseContent; this.md5 = builder.md5; this.dateCreated = builder.dateCreated; @@ -115,14 +107,15 @@ private StorageRecord(final Builder builder) { /** * Create an instance of builder. * - * @return A new instance of builder + * @param the type of id + * @return a new instance of builder */ - public static Builder builder() { - return new Builder(); + public static Builder builder() { + return new Builder<>(); } @Override - public int getId() { + public T getId() { return id; } @@ -157,27 +150,13 @@ public Header[] getResponseHeaders() { } @Override - public String getMimeType() { - return mimeType; - } - - @Override - public Charset getEncoding() { - return encoding; + public ContentType getContentType() { + return contentType; } @Override - public byte[] getResponseContent() throws IOException { - return IOUtils.toByteArray(getStreamResponseContent()); - } - - @Override - public InputStream getStreamResponseContent() throws IOException { - return new BufferedInputStream( - new GZIPInputStream( - new FileInputStream(responseContent) - ) - ); + public byte[] getResponseContent() { + return responseContent; } @Override @@ -185,19 +164,26 @@ public long getDateCreated() { return dateCreated; } + /** + * Get md5 hash of the response content. + * + * @return md5 hash of the response content + */ public String getMD5() { return md5; } /** * A builder for StorageRecord class. + * + * @param the type of id */ - public static class Builder { + public static class Builder { /** * The id of this record. */ - private int id; + private T id; /** * The url for this request. @@ -230,19 +216,14 @@ public static class Builder { private Header[] responseHeaders; /** - * The mime type of the response. - */ - private String mimeType; - - /** - * The encoding of the response. + * The content type of the response. */ - private Charset encoding; + private ContentType contentType; /** * The content of the response. */ - private File responseContent; + private byte[] responseContent; /** * The md5 hash of the content. @@ -260,7 +241,7 @@ public static class Builder { * @param id id for the record * @return this */ - public final Builder setId(final int id) { + public final Builder setId(final T id) { this.id = id; return this; } @@ -332,24 +313,13 @@ public final Builder setResponseHeaders(final Header[] responseHeaders) { } /** - * Sets the response mime type for the record. - * - * @param mimeType mime type of the response - * @return this - */ - public final Builder setMimeType(final String mimeType) { - this.mimeType = mimeType; - return this; - } - - /** - * Sets the response encoding for the record. + * Sets the response content type for the record. * - * @param encoding encoding of the response + * @param contentType content type of the response * @return this */ - public final Builder setEncoding(final Charset encoding) { - this.encoding = encoding; + public final Builder setContentType(final ContentType contentType) { + this.contentType = contentType; return this; } @@ -359,7 +329,7 @@ public final Builder setEncoding(final Charset encoding) { * @param responseContent content of the response * @return this */ - public final Builder setResponseContent(final File responseContent) { + public final Builder setResponseContent(final byte[] responseContent) { this.responseContent = responseContent; return this; } @@ -391,8 +361,8 @@ public final Builder setDateCreated(final long dateCreated) { * * @return an instance of StorageRecord. */ - public final StorageRecord build() { - return new StorageRecord(this); + public final StorageRecord build() { + return new StorageRecord<>(this); } } diff --git a/src/test/java/ai/preferred/venom/fetcher/StorageFetcherTest.java b/src/test/java/ai/preferred/venom/fetcher/StorageFetcherTest.java new file mode 100644 index 0000000..d3a7ebe --- /dev/null +++ b/src/test/java/ai/preferred/venom/fetcher/StorageFetcherTest.java @@ -0,0 +1,261 @@ +/* + * Copyright 2018 Preferred.AI + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.preferred.venom.fetcher; + +import ai.preferred.venom.request.Request; +import ai.preferred.venom.request.VRequest; +import ai.preferred.venom.response.Response; +import ai.preferred.venom.storage.FakeFileManager; +import ai.preferred.venom.storage.Record; +import ai.preferred.venom.storage.StorageException; +import ai.preferred.venom.storage.StorageRecord; +import ai.preferred.venom.validator.Validator; +import org.apache.http.Header; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; + + +public class StorageFetcherTest { + + @Test + public void testTrue() throws Exception { + final String path = "/test-fetch"; + final String url = "http://127.0.0.1/" + path; + final Request request = new VRequest(url); + + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(request.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .build(); + + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(request, record)); + final Validator validator = Validator.ALWAYS_VALID; + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).setValidator(validator).build()) { + fetcher.start(); + final Future responseFuture = fetcher.fetch(request); + final Response response = responseFuture.get(); + Assertions.assertEquals(statusCode, response.getStatusCode()); + Assertions.assertEquals(content, response.getContent()); + Assertions.assertEquals(contentType, response.getContentType()); + Assertions.assertNull(response.getProxy()); + } + + Assertions.assertTrue(fileManager.getClosed()); + } + + @Test + public void testHeadersTrue() throws Exception { + final String path = "/test-headers"; + final String headerKey = "Cookie"; + final String headerValue = "text=json;"; + final String url = "http://127.0.0.1/" + path; + final Request request = new VRequest(url); + + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + final Header[] headers = {new BasicHeader(headerKey, headerValue)}; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(request.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .setResponseHeaders(headers) + .build(); + + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(request, record)); + final Validator validator = Validator.ALWAYS_VALID; + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).setValidator(validator).build()) { + fetcher.start(); + final Future responseFuture = fetcher.fetch(request); + final Response response = responseFuture.get(); + Assertions.assertEquals(statusCode, response.getStatusCode()); + Assertions.assertEquals(content, response.getContent()); + Assertions.assertEquals(contentType, response.getContentType()); + Assertions.assertEquals(headers, response.getHeaders()); + Assertions.assertNull(response.getProxy()); + } + + Assertions.assertTrue(fileManager.getClosed()); + } + + @Test + public void testFetcherHeadersTrue() throws Exception { + final String path = "/fetcher-headers"; + final String headerKey = "Cookie"; + final String headerValue = "text=json;"; + final String url = "http://127.0.0.1/" + path; + final Request submittedRequest = new VRequest(url); + + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + + final Header[] headers = {new BasicHeader(headerKey, headerValue)}; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(submittedRequest.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .setResponseHeaders(headers) + .build(); + + final Map headerMap = Collections.singletonMap(headerKey, headerValue); + final Request request = new VRequest(url, headerMap); + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(request, record)); + final Validator validator = Validator.ALWAYS_VALID; + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).setValidator(validator) + .setHeaders(headerMap).build()) { + final Future responseFuture = fetcher.fetch(submittedRequest); + final Response response = responseFuture.get(); + Assertions.assertEquals(statusCode, response.getStatusCode()); + Assertions.assertEquals(content, response.getContent()); + Assertions.assertEquals(contentType, response.getContentType()); + Assertions.assertEquals(headers, response.getHeaders()); + Assertions.assertNull(response.getProxy()); + } + + Assertions.assertTrue(fileManager.getClosed()); + } + + @Test + public void testNotFound() throws Exception { + final String path = "/not-found"; + final String headerKey = "Cookie"; + final String headerValue = "text=json;"; + final String url = "http://127.0.0.1/" + path; + final Request request = new VRequest(url); + + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + + final Header[] headers = {new BasicHeader(headerKey, headerValue)}; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(request.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .setResponseHeaders(headers) + .build(); + + final Map headerMap = Collections.singletonMap(headerKey, headerValue); + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(request, record)); + final Validator validator = Validator.ALWAYS_VALID; + final AtomicBoolean thrown = new AtomicBoolean(false); + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).setValidator(validator) + .setHeaders(headerMap).build()) { + final Future responseFuture = fetcher.fetch(request); + Assertions.assertTrue(responseFuture.isCancelled()); + try { + final Response response = responseFuture.get(); + } catch (CancellationException e) { + thrown.set(true); + } catch (InterruptedException | ExecutionException e) { + Assertions.fail("Wrong exception"); + } + } + + Assertions.assertTrue(thrown.get(), "CancellationException not thrown."); + Assertions.assertTrue(fileManager.getClosed()); + } + + @Test + public void testFailure() throws Exception { + final String path = "/test-failure"; + final String url = "http://127.0.0.1/" + path; + final Request request = new VRequest(url); + + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(request.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .build(); + + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(null, record)); + final Validator validator = Validator.ALWAYS_VALID; + final AtomicBoolean thrown = new AtomicBoolean(false); + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).setValidator(validator).build()) { + final Future responseFuture = fetcher.fetch(request); + try { + final Response response = responseFuture.get(); + } catch (InterruptedException | ExecutionException e) { + Assertions.assertTrue(e.getCause() instanceof StorageException); + thrown.set(true); + } + } + + Assertions.assertTrue(thrown.get(), "StorageException not thrown."); + Assertions.assertTrue(fileManager.getClosed()); + } + + @Test + public void testValidation() throws Exception { + final String path = "/test-validation"; + final String url = "http://127.0.0.1/" + path; + final Request request = new VRequest(url); + + final int statusCode = 500; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + final Record record = StorageRecord.builder() + .setUrl(url) + .setRequestMethod(request.getMethod()) + .setStatusCode(statusCode) + .setResponseContent(content) + .setContentType(contentType) + .build(); + + final FakeFileManager fileManager = new FakeFileManager(Collections.singletonMap(request, record)); + final AtomicBoolean thrown = new AtomicBoolean(false); + try (final Fetcher fetcher = StorageFetcher.builder(fileManager).build()) { + final Future responseFuture = fetcher.fetch(request); + try { + final Response response = responseFuture.get(); + } catch (InterruptedException | ExecutionException e) { + Assertions.assertTrue(e.getCause() instanceof ValidationException); + thrown.set(true); + } + } + + Assertions.assertTrue(thrown.get(), "ValidationException not thrown."); + Assertions.assertTrue(fileManager.getClosed()); + } +} \ No newline at end of file diff --git a/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java b/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java new file mode 100644 index 0000000..12e83a7 --- /dev/null +++ b/src/test/java/ai/preferred/venom/storage/DummyFileManagerTest.java @@ -0,0 +1,108 @@ +/* + * Copyright 2018 Preferred.AI + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.preferred.venom.storage; + +import ai.preferred.venom.request.Request; +import ai.preferred.venom.request.VRequest; +import ai.preferred.venom.response.BaseResponse; +import ai.preferred.venom.response.Response; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.http.Header; +import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; +import org.junit.jupiter.api.*; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class DummyFileManagerTest { + + private DummyFileManager fileManager; + + private Path storage; + + @BeforeAll + void setUp() throws IOException { + storage = Files.createTempDirectory("test_storage_directory"); + fileManager = new DummyFileManager(storage.toFile()); + } + + @AfterAll + void tearDown() throws Exception { + Exception cached = null; + try { + fileManager.close(); + } catch (final Exception e) { + cached = e; + } + + try { + //noinspection ResultOfMethodCallIgnored + Files.walk(storage) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } catch (final IOException e) { + if (cached != null) { + cached.addSuppressed(e); + } else { + cached = e; + } + } + + if (cached != null) { + throw cached; + } + } + + @Test + void testPut() throws StorageException { + final int statusCode = 200; + final String url = "https://preferred.ai/"; + final byte[] content = "This is test data.".getBytes(); + final ContentType contentType = ContentType.create("text/html", StandardCharsets.UTF_8); + final Header[] headers = {}; + final HttpHost proxy = null; + + final Request request = new VRequest(url); + final Response response = new BaseResponse(statusCode, url, content, contentType, headers, proxy); + + final String md5 = DigestUtils.md5Hex(content); + final String subDirName = md5.substring(0, 2); + + final String path = fileManager.put(request, response); + final String expectedPath = new File(new File(storage.toFile(), subDirName), md5).toString() + ".html"; + Assertions.assertEquals(expectedPath, path); + } + + @Test + void testGetId() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> fileManager.get(new Object())); + } + + @Test + void testGetRequest() { + final String url = "https://preferred.ai/"; + final Request request = new VRequest(url); + Assertions.assertThrows(UnsupportedOperationException.class, () -> fileManager.get(request)); + } +} diff --git a/src/test/java/ai/preferred/venom/storage/FakeFileManager.java b/src/test/java/ai/preferred/venom/storage/FakeFileManager.java new file mode 100644 index 0000000..66d26be --- /dev/null +++ b/src/test/java/ai/preferred/venom/storage/FakeFileManager.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Preferred.AI + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.preferred.venom.storage; + +import ai.preferred.venom.fetcher.Callback; +import ai.preferred.venom.request.Request; +import ai.preferred.venom.response.Response; + +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +public class FakeFileManager implements FileManager { + + private final Callback callback; + + private final Map requestRecordMap; + + private final AtomicBoolean closed = new AtomicBoolean(false); + + public FakeFileManager() { + this(Collections.emptyMap()); + } + + public FakeFileManager(final Map requestRecordMap) { + this.callback = new FileManagerCallback(this); + this.requestRecordMap = requestRecordMap; + } + + @Override + public @NotNull Callback getCallback() { + return callback; + } + + @Override + public @NotNull String put(@NotNull Request request, @NotNull Response response) { + return "true"; + } + + @Override + public @NotNull Record get(Object id) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public @NotNull Record get(@NotNull Request request) throws StorageException { + for (Map.Entry requestRecordEntry : requestRecordMap.entrySet()) { + final Request trueRequest = requestRecordEntry.getKey(); + + if (trueRequest == null) { + throw new StorageException("Throw code captured."); + } + + if (trueRequest.getUrl().equals(request.getUrl()) + && trueRequest.getMethod() == trueRequest.getMethod() + && trueRequest.getHeaders().equals(request.getHeaders())) { + if ((trueRequest.getBody() != null && request.getBody() != null + && trueRequest.getBody().equals(request.getBody())) + || (trueRequest.getBody() == null && request.getBody() == null)) { + return requestRecordEntry.getValue(); + } + } + } + return null; + } + + @Override + public void close() { + closed.set(true); + } + + public boolean getClosed() { + return closed.get(); + } +} diff --git a/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java b/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java index ad66b9d..e85747c 100644 --- a/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java +++ b/src/test/java/ai/preferred/venom/storage/MysqlFileManagerTest.java @@ -42,7 +42,7 @@ public class MysqlFileManagerTest { private DB db; - private FileManager fileManager; + private MysqlFileManager fileManager; private Path storage; @BeforeAll @@ -65,6 +65,7 @@ void tearDown() throws Exception { } try { + //noinspection ResultOfMethodCallIgnored Files.walk(storage) .sorted(Comparator.reverseOrder()) .map(Path::toFile) diff --git a/src/test/java/ai/preferred/venom/storage/StorageRecordTest.java b/src/test/java/ai/preferred/venom/storage/StorageRecordTest.java new file mode 100644 index 0000000..3b7ece8 --- /dev/null +++ b/src/test/java/ai/preferred/venom/storage/StorageRecordTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Preferred.AI + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.preferred.venom.storage; + +import ai.preferred.venom.request.Request; +import ai.preferred.venom.request.VRequest; +import org.apache.http.Header; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicHeader; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; + +public class StorageRecordTest { + + @Test + void test() { + final String path = "/test-headers"; + final String headerKey = "Cookie"; + final String headerValue = "text=json;"; + final String url = "http://127.0.0.1/" + path; + final Map headerMap = Collections.singletonMap(headerKey, headerValue); + final Request request = new VRequest(url, headerMap); + + final Object id = new Object(); + final int statusCode = 200; + final byte[] content = "This is a test".getBytes(); + final ContentType contentType = ContentType.TEXT_PLAIN; + final Header[] headers = {new BasicHeader(headerKey, headerValue)}; + final long dateCreated = 0; + final String md5 = "md5"; + final Map body = Collections.singletonMap("body", "body"); + + final StorageRecord storageRecord = StorageRecord.builder() + .setId(id) + .setResponseHeaders(headers) + .setContentType(contentType) + .setResponseContent(content) + .setStatusCode(statusCode) + .setRequestMethod(request.getMethod()) + .setUrl(url) + .setDateCreated(dateCreated) + .setMD5(md5) + .setRequestHeaders(headerMap) + .setRequestBody(body) + .build(); + + Assertions.assertEquals(id, storageRecord.getId()); + Assertions.assertEquals(headers, storageRecord.getResponseHeaders()); + Assertions.assertEquals(contentType, storageRecord.getContentType()); + Assertions.assertEquals(content, storageRecord.getResponseContent()); + Assertions.assertEquals(statusCode, storageRecord.getStatusCode()); + Assertions.assertEquals(request.getMethod(), storageRecord.getRequestMethod()); + Assertions.assertEquals(url, storageRecord.getURL()); + Assertions.assertEquals(dateCreated, storageRecord.getDateCreated()); + Assertions.assertEquals(md5, storageRecord.getMD5()); + Assertions.assertEquals(headerMap, storageRecord.getRequestHeaders()); + Assertions.assertEquals(body, storageRecord.getRequestBody()); + } + +}