Skip to content

Commit

Permalink
Changed the REST API and actions for scraping [#438]
Browse files Browse the repository at this point in the history 
 - Renamed ComicVineScraperController => ScrapingController
 - Added the ScrapingService class.
 - ScrapingController delegates everything to ScrapingService.
 - Made ScrapingAdaptor more generic.
 - Added a new API to the ScrapingAdaptor for getting issue details.
  • Loading branch information
@mcpierce
mcpierce committed Aug 1, 2020
1 parent 7ab6c9f commit 43e235d
Show file tree
Hide file tree
Showing 17 changed files with 1,384 additions and 1,128 deletions.
32 changes: 8 additions & 24 deletions .../scraping/ComicVineScraperController.java → ...ntroller/scraping/ScrapingController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,24 @@
import org.comixedproject.net.GetScrapingIssueRequest;
import org.comixedproject.net.GetVolumesRequest;
import org.comixedproject.scrapers.ScrapingException;
import org.comixedproject.scrapers.comicvine.adaptors.ComicVineScrapingAdaptor;
import org.comixedproject.scrapers.model.ScrapingIssue;
import org.comixedproject.scrapers.model.ScrapingVolume;
import org.comixedproject.service.comic.ComicException;
import org.comixedproject.service.comic.ComicService;
import org.comixedproject.service.scraping.ScrapingService;
import org.comixedproject.views.View;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;

/**
* <code>ComicVineScraperController</code> processes REST APIs relating to scraping comics.
* <code>ScrapingController</code> processes REST APIs relating to scraping comics.
*
* @author Darryl L. Pierce
*/
@RestController
@RequestMapping("/api/scraping")
@Log4j2
public class ComicVineScraperController {
@Autowired private ComicVineScrapingAdaptor scrapingAdaptor;
@Autowired private ComicService comicService;
public class ScrapingController {
@Autowired private ScrapingService scrapingService;

/**
* Retrieves the minimal {@link ScrapingIssue} for the specified issue of the given volume.
Expand All @@ -73,7 +70,7 @@ public ScrapingIssue queryForIssue(
"Preparing to retrieve issue={} for volume={} (skipCache={})", issue, volume, skipCache);

try {
return this.scrapingAdaptor.getIssue(apiKey, volume, issue, skipCache);
return this.scrapingService.getIssue(apiKey, volume, issue, skipCache);
} catch (ScrapingException error) {
throw new ComiXedControllerException("Failed to get single scraping issue", error);
}
Expand Down Expand Up @@ -105,7 +102,7 @@ public List<ScrapingVolume> queryForVolumes(@RequestBody() final GetVolumesReque

try {
final List<ScrapingVolume> result =
this.scrapingAdaptor.getVolumes(apiKey, series, maxRecords, skipCache);
this.scrapingService.getVolumes(apiKey, series, maxRecords, skipCache);

log.debug("Returning {} volume{}", result.size(), result.size() == 1 ? "" : "s");

Expand All @@ -131,30 +128,17 @@ public List<ScrapingVolume> queryForVolumes(@RequestBody() final GetVolumesReque
@JsonView(View.ComicDetails.class)
public Comic scrapeAndSaveComicDetails(
@PathVariable("comicId") final Long comicId,
@PathVariable("issueId") final String issueId,
@PathVariable("issueId") final Integer issueId,
@RequestBody() final ComicScrapeRequest request)
throws ComiXedControllerException {
boolean skipCache = request.getSkipCache();
String apiKey = request.getApiKey();

log.info("Scraping code: id={} issue id={} (skip cache={})", comicId, issueId, apiKey);

log.debug("Loading comic");
Comic comic = null;
try {
comic = this.comicService.getComic(comicId);
} catch (ComicException error) {
throw new ComiXedControllerException("Failed to load comic", error);
}

try {
log.debug("Scraping comic details");
this.scrapingAdaptor.scrapeComic(apiKey, issueId, skipCache, comic);

log.debug("Saving updated comic");
this.comicService.save(comic);

return comic;
return this.scrapingService.scrapeComic(apiKey, comicId, issueId, skipCache);
} catch (ScrapingException error) {
throw new ComiXedControllerException("Failed to scrape comic", error);
}
Expand Down
70 changes: 27 additions & 43 deletions ...aping/ComicVineScraperControllerTest.java → ...ller/scraping/ScrapingControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,10 @@
import org.comixedproject.net.GetScrapingIssueRequest;
import org.comixedproject.net.GetVolumesRequest;
import org.comixedproject.scrapers.ScrapingException;
import org.comixedproject.scrapers.comicvine.adaptors.ComicVineScrapingAdaptor;
import org.comixedproject.scrapers.model.ScrapingIssue;
import org.comixedproject.scrapers.model.ScrapingVolume;
import org.comixedproject.service.comic.ComicException;
import org.comixedproject.service.comic.ComicService;
import org.comixedproject.service.scraping.ScrapingService;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
Expand All @@ -43,44 +42,43 @@

@RunWith(MockitoJUnitRunner.class)
@SpringBootTest
public class ComicVineScraperControllerTest {
public class ScrapingControllerTest {
private static final String TEST_API_KEY = "12345";
private static final String TEST_SERIES_NAME = "Awesome Comic";
private static final Integer TEST_MAX_RECORDS = 37;
private static final Integer TEST_VOLUME = 2018;
private static final String TEST_ISSUE_NUMBER = "15";
private static final long TEST_COMIC_ID = 213L;
private static final String TEST_ISSUE_ID = "48132";
private static final Integer TEST_ISSUE_ID = 48132;
private static final boolean TEST_SKIP_CACHE = true;

@InjectMocks private ComicVineScraperController controller;
@Mock private ComicVineScrapingAdaptor scrapingAdaptor;
@InjectMocks private ScrapingController controller;
@Mock private ScrapingService scrapingService;
@Mock private List<ScrapingVolume> comicVolumeList;
@Mock private ScrapingIssue comicIssue;
@Mock private ComicService comicService;
@Mock private Comic comic;

@Test(expected = ComiXedControllerException.class)
public void testQueryForVolumesAdaptorRaisesException()
throws ScrapingException, ComiXedControllerException {
Mockito.when(
scrapingAdaptor.getVolumes(
scrapingService.getVolumes(
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenThrow(ScrapingException.class);

try {
controller.queryForVolumes(
new GetVolumesRequest(TEST_API_KEY, TEST_SERIES_NAME, TEST_MAX_RECORDS, false));
} finally {
Mockito.verify(scrapingAdaptor, Mockito.times(1))
Mockito.verify(scrapingService, Mockito.times(1))
.getVolumes(TEST_API_KEY, TEST_SERIES_NAME, TEST_MAX_RECORDS, false);
}
}

@Test
public void testQueryForVolumes() throws ComiXedControllerException, ScrapingException {
Mockito.when(
scrapingAdaptor.getVolumes(
scrapingService.getVolumes(
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenReturn(comicVolumeList);

Expand All @@ -90,14 +88,14 @@ public void testQueryForVolumes() throws ComiXedControllerException, ScrapingExc

assertSame(comicVolumeList, result);

Mockito.verify(scrapingAdaptor, Mockito.times(1))
Mockito.verify(scrapingService, Mockito.times(1))
.getVolumes(TEST_API_KEY, TEST_SERIES_NAME, TEST_MAX_RECORDS, false);
}

@Test
public void testQueryForVolumesSkipCache() throws ScrapingException, ComiXedControllerException {
Mockito.when(
scrapingAdaptor.getVolumes(
scrapingService.getVolumes(
Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenReturn(comicVolumeList);

Expand All @@ -107,15 +105,15 @@ public void testQueryForVolumesSkipCache() throws ScrapingException, ComiXedCont

assertSame(comicVolumeList, result);

Mockito.verify(scrapingAdaptor, Mockito.times(1))
Mockito.verify(scrapingService, Mockito.times(1))
.getVolumes(TEST_API_KEY, TEST_SERIES_NAME, TEST_MAX_RECORDS, true);
}

@Test(expected = ComiXedControllerException.class)
public void testQueryForIssueAdaptorRaisesException()
throws ScrapingException, ComiXedControllerException {
Mockito.when(
scrapingAdaptor.getIssue(
scrapingService.getIssue(
Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()))
.thenThrow(ScrapingException.class);

Expand All @@ -124,15 +122,15 @@ public void testQueryForIssueAdaptorRaisesException()
TEST_VOLUME,
new GetScrapingIssueRequest(TEST_API_KEY, TEST_SKIP_CACHE, TEST_ISSUE_NUMBER));
} finally {
Mockito.verify(scrapingAdaptor, Mockito.times(1))
Mockito.verify(scrapingService, Mockito.times(1))
.getIssue(TEST_API_KEY, TEST_VOLUME, TEST_ISSUE_NUMBER, TEST_SKIP_CACHE);
}
}

@Test
public void testQueryForIssue() throws ScrapingException, ComiXedControllerException {
Mockito.when(
scrapingAdaptor.getIssue(
scrapingService.getIssue(
Mockito.anyString(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()))
.thenReturn(comicIssue);

Expand All @@ -144,48 +142,34 @@ public void testQueryForIssue() throws ScrapingException, ComiXedControllerExcep
assertNotNull(result);
assertSame(comicIssue, result);

Mockito.verify(scrapingAdaptor, Mockito.times(1))
Mockito.verify(scrapingService, Mockito.times(1))
.getIssue(TEST_API_KEY, TEST_VOLUME, TEST_ISSUE_NUMBER, TEST_SKIP_CACHE);
}

@Test(expected = ComiXedControllerException.class)
public void testScrapeAndSaveComicDetailsNoSuchComic()
throws ComicException, ComiXedControllerException {
Mockito.when(comicService.getComic(Mockito.anyLong())).thenThrow(ComicException.class);

try {
controller.scrapeAndSaveComicDetails(
TEST_COMIC_ID, TEST_ISSUE_ID, new ComicScrapeRequest(TEST_API_KEY, TEST_SKIP_CACHE));
} finally {
Mockito.verify(comicService, Mockito.times(1)).getComic(TEST_COMIC_ID);
}
}

@Test(expected = ComiXedControllerException.class)
public void testScrapeAndSaveComicDetailsScrapingAdaptorRaisesException()
throws ComicException, ScrapingException, ComiXedControllerException {
Mockito.when(comicService.getComic(Mockito.anyLong())).thenReturn(comic);
Mockito.doThrow(ScrapingException.class)
.when(scrapingAdaptor)
.scrapeComic(
Mockito.anyString(),
Mockito.anyString(),
Mockito.anyBoolean(),
Mockito.any(Comic.class));
Mockito.when(
scrapingService.scrapeComic(
Mockito.anyString(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenThrow(ScrapingException.class);

try {
controller.scrapeAndSaveComicDetails(
TEST_COMIC_ID, TEST_ISSUE_ID, new ComicScrapeRequest(TEST_API_KEY, TEST_SKIP_CACHE));
} finally {
Mockito.verify(scrapingAdaptor, Mockito.times(1))
.scrapeComic(TEST_API_KEY, TEST_ISSUE_ID, TEST_SKIP_CACHE, comic);
Mockito.verify(scrapingService, Mockito.times(1))
.scrapeComic(TEST_API_KEY, TEST_COMIC_ID, TEST_ISSUE_ID, TEST_SKIP_CACHE);
}
}

@Test
public void testScrapeAndSaveComicDetails()
throws ComicException, ScrapingException, ComiXedControllerException {
Mockito.when(comicService.getComic(Mockito.anyLong())).thenReturn(comic);
Mockito.when(
scrapingService.scrapeComic(
Mockito.anyString(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyBoolean()))
.thenReturn(comic);

Comic result =
controller.scrapeAndSaveComicDetails(
Expand All @@ -194,7 +178,7 @@ public void testScrapeAndSaveComicDetails()
assertNotNull(result);
assertSame(comic, result);

Mockito.verify(scrapingAdaptor, Mockito.times(1))
.scrapeComic(TEST_API_KEY, TEST_ISSUE_ID, TEST_SKIP_CACHE, comic);
Mockito.verify(scrapingService, Mockito.times(1))
.scrapeComic(TEST_API_KEY, TEST_COMIC_ID, TEST_ISSUE_ID, TEST_SKIP_CACHE);
}
}
2 changes: 1 addition & 1 deletion comixed-scrapers/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<dependencies>
<dependency>
<groupId>org.comixedproject</groupId>
<artifactId>comixed-services</artifactId>
<artifactId>comixed-model</artifactId>
<version>${comixed.version}</version>
</dependency>
<dependency>
Expand Down

0 comments on commit 43e235d

Please sign in to comment.