Added the scraping feature [#559]

* Added an action to load scraping volumes. * Added an action to load a scraping issue. * Added an action to scrape a comic.
comixed · Dec 25, 2020 · 62be5c4 · 62be5c4
1 parent 9a37c97
commit 62be5c4
Show file tree

Hide file tree

Showing 30 changed files with 1,528 additions and 109 deletions.
diff --git a/...ntroller/scraping/ScrapingController.java → ...ontroller/library/ScrapingController.java b/...ntroller/scraping/ScrapingController.java → ...ontroller/library/ScrapingController.java
@@ -16,54 +16,60 @@
  * along with this program. If not, see <http://www.gnu.org/licenses>
  */
 
-package org.comixedproject.controller.scraping;
+package org.comixedproject.controller.library;
 
 import com.fasterxml.jackson.annotation.JsonView;
 import java.util.List;
 import lombok.extern.log4j.Log4j2;
 import org.comixedproject.auditlog.AuditableEndpoint;
 import org.comixedproject.model.comic.Comic;
-import org.comixedproject.model.net.ComicScrapeRequest;
-import org.comixedproject.model.net.GetScrapingIssueRequest;
-import org.comixedproject.model.net.GetVolumesRequest;
+import org.comixedproject.model.net.library.LoadScrapingIssueRequest;
+import org.comixedproject.model.net.library.LoadScrapingVolumesRequest;
+import org.comixedproject.model.net.library.ScrapeComicRequest;
 import org.comixedproject.scrapers.ScrapingException;
 import org.comixedproject.scrapers.model.ScrapingIssue;
 import org.comixedproject.scrapers.model.ScrapingVolume;
 import org.comixedproject.service.scraping.ScrapingService;
 import org.comixedproject.views.View;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RestController;
 
 /**
  * <code>ScrapingController</code> processes REST APIs relating to scraping comics.
  *
  * @author Darryl L. Pierce
  */
 @RestController
-@RequestMapping("/api/scraping")
 @Log4j2
 public class ScrapingController {
   @Autowired private ScrapingService scrapingService;
 
   /**
-   * Retrieves the minimal {@link ScrapingIssue} for the specified issue of the given volume.
+   * Retrieves a single {@link ScrapingIssue} for the specified issue of the given volume and issue
+   * number.
    *
    * @param volume the volume id
+   * @param issue the issue number
    * @param request the request body
    * @return the issue
    * @throws ScrapingException if an error occurs
    */
   @PostMapping(
-      value = "/volumes/{volume}/issues",
+      value = "/api/scraping/volumes/{volumeId}/issues/{issueId}",
       produces = MediaType.APPLICATION_JSON_VALUE,
       consumes = MediaType.APPLICATION_JSON_VALUE)
+  @PreAuthorize("hasRole('ADMIN')")
   @AuditableEndpoint
-  public ScrapingIssue queryForIssue(
-      @PathVariable("volume") final Integer volume,
-      @RequestBody() final GetScrapingIssueRequest request)
+  public ScrapingIssue loadScrapingIssue(
+      @PathVariable("volumeId") final Integer volume,
+      @PathVariable("issueId") final String issue,
+      @RequestBody() final LoadScrapingIssueRequest request)
       throws ScrapingException {
-    String issue = request.getIssueNumber();
     boolean skipCache = request.isSkipCache();
     String apiKey = request.getApiKey();
 
@@ -81,16 +87,17 @@ public ScrapingIssue queryForIssue(
    * @throws ScrapingException if an error occurs
    */
   @PostMapping(
-      value = "/volumes",
+      value = "/api/scraping/volumes",
       produces = MediaType.APPLICATION_JSON_VALUE,
       consumes = MediaType.APPLICATION_JSON_VALUE)
+  @PreAuthorize("hasRole('ADMIN')")
   @AuditableEndpoint
-  public List<ScrapingVolume> queryForVolumes(@RequestBody() final GetVolumesRequest request)
-      throws ScrapingException {
+  public List<ScrapingVolume> loadScrapingVolumes(
+      @RequestBody() final LoadScrapingVolumesRequest request) throws ScrapingException {
     String apiKey = request.getApiKey();
-    boolean skipCache = request.getSkipCache();
     String series = request.getSeries();
-    final Integer maxRecords = request.getMaxRecords();
+    final int maxRecords = request.getMaxRecords();
+    boolean skipCache = request.getSkipCache();
 
     log.info(
         "Getting volumes: series={} (max records={}) {}",
@@ -105,22 +112,22 @@ public List<ScrapingVolume> queryForVolumes(@RequestBody() final GetVolumesReque
    * Scrapes a single {@link Comic} using the specified source issue.
    *
    * @param comicId the comic id
-   * @param issueId the issue id
    * @param request the request body
    * @return the scraped and updated {@link Comic}
    * @throws ScrapingException if an error occurs
    */
   @PostMapping(
-      value = "/comics/{comicId}/issue/{issueId}",
+      value = "/api/scraping/comics/{comicId}",
       produces = MediaType.APPLICATION_JSON_VALUE,
       consumes = MediaType.APPLICATION_JSON_VALUE)
+  @PreAuthorize("hasRole('ADMIN')")
+  @AuditableEndpoint
   @JsonView(View.ComicDetailsView.class)
-  public Comic scrapeAndSaveComicDetails(
-      @PathVariable("comicId") final Long comicId,
-      @PathVariable("issueId") final Integer issueId,
-      @RequestBody() final ComicScrapeRequest request)
+  public Comic scrapeComic(
+      @PathVariable("comicId") final Long comicId, @RequestBody() final ScrapeComicRequest request)
       throws ScrapingException {
     boolean skipCache = request.getSkipCache();
+    Integer issueId = request.getIssueId();
     String apiKey = request.getApiKey();
 
     log.info("Scraping code: id={} issue id={} (skip cache={})", comicId, issueId, apiKey);

diff --git a/...project/model/net/ComicScrapeRequest.java → ...net/library/LoadScrapingIssueRequest.java b/...project/model/net/ComicScrapeRequest.java → ...net/library/LoadScrapingIssueRequest.java
@@ -16,27 +16,26 @@
  * along with this program. If not, see <http://www.gnu.org/licenses>
  */
 
-package org.comixedproject.model.net;
+package org.comixedproject.model.net.library;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
-import lombok.Setter;
+import lombok.NoArgsConstructor;
 
 /**
- * <code>ComicScrapeRequest</code> hols the details for scraping a single comic issue.
+ * <code>LoadScrapingIssueRequest</code> represents the request to load a single scraping issue.
  *
  * @author Darryl L. Pierce
  */
+@NoArgsConstructor
 @AllArgsConstructor
-public class ComicScrapeRequest {
+public class LoadScrapingIssueRequest {
   @JsonProperty("apiKey")
   @Getter
-  @Setter
-  private final String apiKey;
+  private String apiKey;
 
   @JsonProperty("skipCache")
   @Getter
-  @Setter
-  private final Boolean skipCache;
+  private boolean skipCache;
 }
diff --git a/...dproject/model/net/GetVolumesRequest.java → ...t/library/LoadScrapingVolumesRequest.java b/...dproject/model/net/GetVolumesRequest.java → ...t/library/LoadScrapingVolumesRequest.java
@@ -16,33 +16,35 @@
  * along with this program. If not, see <http://www.gnu.org/licenses>
  */
 
-package org.comixedproject.model.net;
+package org.comixedproject.model.net.library;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 
 /**
- * <code>GetVolumesRequest</code> represents the payload for requesting a set of volumes for a given
- * series.
+ * <code>LoadScrapingVolumesRequest</code> represents the payload for requesting a set of volumes
+ * for a given series.
  *
  * @author Darryl L. Pierce
  */
 @AllArgsConstructor
-public class GetVolumesRequest {
+@NoArgsConstructor
+public class LoadScrapingVolumesRequest {
   @JsonProperty("apiKey")
   @Getter
-  private final String apiKey;
+  private String apiKey;
 
   @JsonProperty("series")
   @Getter
-  private final String series;
+  private String series;
 
   @JsonProperty("maxRecords")
   @Getter
-  private final Integer maxRecords;
+  private Integer maxRecords;
 
   @JsonProperty("skipCache")
   @Getter
-  private final Boolean skipCache;
+  private Boolean skipCache;
 }
diff --git a/...ct/model/net/GetScrapingIssueRequest.java → ...model/net/library/ScrapeComicRequest.java b/...ct/model/net/GetScrapingIssueRequest.java → ...model/net/library/ScrapeComicRequest.java
@@ -16,38 +16,34 @@
  * along with this program. If not, see <http://www.gnu.org/licenses>
  */
 
-package org.comixedproject.model.net;
+package org.comixedproject.model.net.library;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
 
-public class GetScrapingIssueRequest {
+/**
+ * <code>ScrapeComicRequest</code> hols the details for scraping a single comic issue.
+ *
+ * @author Darryl L. Pierce
+ */
+@AllArgsConstructor
+@NoArgsConstructor
+public class ScrapeComicRequest {
   @JsonProperty("apiKey")
+  @Getter
+  @Setter
   private String apiKey;
 
-  @JsonProperty("skipCache")
-  private boolean skipCache;
-
-  @JsonProperty("issueNumber")
-  private String issueNumber;
-
-  public GetScrapingIssueRequest() {}
+  @JsonProperty("issueId")
+  @Getter
+  @Setter
+  private Integer issueId;
 
-  public GetScrapingIssueRequest(
-      final String apiKey, final boolean skipCache, final String issueNumber) {
-    this.apiKey = apiKey;
-    this.skipCache = skipCache;
-    this.issueNumber = issueNumber;
-  }
-
-  public String getApiKey() {
-    return apiKey;
-  }
-
-  public boolean isSkipCache() {
-    return skipCache;
-  }
-
-  public String getIssueNumber() {
-    return issueNumber;
-  }
+  @JsonProperty("skipCache")
+  @Getter
+  @Setter
+  private Boolean skipCache;
 }