Skip to content

Commit

Permalink
switch WasapiDownloader.downloadSelectedWarcs() to use pagedJsonQuery…
Browse files Browse the repository at this point in the history 
… instead of jsonQuery

fix tests accordingly, slightly tighten up mock WasapiCrawlSelector constructor arg expectations
  • Loading branch information
@jmartin-sul
jmartin-sul committed May 31, 2017
1 parent 4913d7f commit 7e22eff
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 27 deletions.
6 changes: 3 additions & 3 deletions src/edu/stanford/dlss/was/WasapiDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ WasapiConnection getWasapiConn() throws IOException {
// package level method for testing
void downloadSelectedWarcs() throws IOException {
// System.out.println("DEBUG: about to request " + getFileSetRequestUrl());
WasapiResponse wasapiResp = getWasapiConn().jsonQuery(getFileSetRequestUrl());
List<WasapiResponse> wasapiRespList = getWasapiConn().pagedJsonQuery(getFileSetRequestUrl());
// System.out.println(wasapiResp.toString());

if (wasapiResp != null) {
WasapiCrawlSelector crawlSelector = new WasapiCrawlSelector(wasapiResp.getFiles());
if (wasapiRespList != null && wasapiRespList.get(0) != null) {
WasapiCrawlSelector crawlSelector = new WasapiCrawlSelector(wasapiRespList);
for (Integer crawlId : desiredCrawlIds(crawlSelector)) {
for (WasapiFile file : crawlSelector.getFilesForCrawl(crawlId)) {
// TODO: make a separate method for downloading individual file?
Expand Down
56 changes: 32 additions & 24 deletions test/edu/stanford/dlss/was/TestWasapiDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,77 +46,84 @@ public void main_withHelp_canExecuteWithoutCrashing() throws SettingsLoadExcepti
public void main_executesFileSetRequest_usesAllAppropArgsSettings() throws Exception {
String[] args = {"--collectionId", "123", "--jobId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14", "--username=Fred" };
WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(null);
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(null);
WasapiDownloader downloaderSpy = PowerMockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, args));
PowerMockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();
PowerMockito.whenNew(WasapiDownloader.class).withAnyArguments().thenReturn(downloaderSpy);

WasapiDownloader.main(args);
verify(mockConn).jsonQuery(ArgumentMatchers.contains("collection=123"));
verify(mockConn).jsonQuery(ArgumentMatchers.contains("crawl=456"));
verify(mockConn).jsonQuery(ArgumentMatchers.contains("crawl-start-after=2014-03-14"));
verify(mockConn).jsonQuery(ArgumentMatchers.contains("crawl-start-before=2017-03-14"));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("collection=123"));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("crawl=456"));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-after=2014-03-14"));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-before=2017-03-14"));
// username is used in login request
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("username=Fred"));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("username=Fred"));
// output directory is not part of wasapi request
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains(WasapiDownloaderSettings.OUTPUT_BASE_DIR_PARAM_NAME));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains(WasapiDownloaderSettings.OUTPUT_BASE_DIR_PARAM_NAME));
}

@Test
public void main_executesFileSetRequest_onlyUsesArgsSettings() throws Exception {
String[] args = {"--collectionId", "123" };
WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(null);
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(null);
WasapiDownloader downloaderSpy = PowerMockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, args));
PowerMockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();
PowerMockito.whenNew(WasapiDownloader.class).withAnyArguments().thenReturn(downloaderSpy);

WasapiDownloader.main(args);
verify(mockConn).jsonQuery(ArgumentMatchers.contains("collection=123"));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl="));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl-start-after="));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl-start-before="));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("collection=123"));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl="));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-after="));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-before="));
}

@Test
public void main_singleFileDownload_onlyUsesFilename() throws Exception {
String[] args = {"--collectionId", "123", "--filename", "ARCHIVEIT-5425-MONTHLY-JOB302671-20170526114117181-00049.warc.gz" };
WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(null);
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(null);
WasapiDownloader downloaderSpy = PowerMockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, args));
PowerMockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();
PowerMockito.whenNew(WasapiDownloader.class).withAnyArguments().thenReturn(downloaderSpy);

WasapiDownloader.main(args);
verify(mockConn).jsonQuery(ArgumentMatchers.contains("filename=ARCHIVEIT-5425-MONTHLY-JOB302671-20170526114117181-00049.warc.gz"));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl="));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl-start-after="));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("crawl-start-before="));
verify(mockConn, Mockito.never()).jsonQuery(ArgumentMatchers.contains("collection="));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.contains("filename=ARCHIVEIT-5425-MONTHLY-JOB302671-20170526114117181-00049.warc.gz"));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl="));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-after="));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("crawl-start-before="));
verify(mockConn, Mockito.never()).pagedJsonQuery(ArgumentMatchers.contains("collection="));
}

@Test
public void downloadSelectedWarcs_requestsFileSetResponse() throws Exception {
WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(null);
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(null);
WasapiDownloader downloaderSpy = Mockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, null));
Mockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();

downloaderSpy.downloadSelectedWarcs();
WasapiDownloaderSettings mySettings = new WasapiDownloaderSettings(WasapiDownloader.SETTINGS_FILE_LOCATION, null);
verify(mockConn).jsonQuery(ArgumentMatchers.startsWith(mySettings.baseUrlString()));
verify(mockConn).pagedJsonQuery(ArgumentMatchers.startsWith(mySettings.baseUrlString()));
}

private List<WasapiResponse> getWasapiRespList() {
List<WasapiResponse> wasapiRespList = new ArrayList<WasapiResponse>();
wasapiRespList.add(new WasapiResponse());
return wasapiRespList;
}

@Test
public void downloadSelectedWarcs_usesCrawlSelector() throws Exception {
WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(new WasapiResponse());
List<WasapiResponse> wasapiRespList = getWasapiRespList();
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(wasapiRespList);

WasapiCrawlSelector mockCrawlSelector = PowerMockito.mock(WasapiCrawlSelector.class);
List<Integer> desiredCrawlIds = new ArrayList<Integer>();
desiredCrawlIds.add(Integer.valueOf("666"));
PowerMockito.when(mockCrawlSelector.getSelectedCrawlIds(0)).thenReturn(desiredCrawlIds);
PowerMockito.whenNew(WasapiCrawlSelector.class).withAnyArguments().thenReturn(mockCrawlSelector);
PowerMockito.whenNew(WasapiCrawlSelector.class).withArguments(wasapiRespList).thenReturn(mockCrawlSelector);

WasapiDownloader downloaderSpy = Mockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, null));
Mockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();
Expand All @@ -133,10 +140,11 @@ public void downloadSelectedWarcs_byJobIdLowerBound() throws Exception {
String[] args = { "--jobIdLowerBound=" + argValue };

WasapiCrawlSelector mockCrawlSelector = PowerMockito.mock(WasapiCrawlSelector.class);
PowerMockito.whenNew(WasapiCrawlSelector.class).withAnyArguments().thenReturn(mockCrawlSelector);
List<WasapiResponse> wasapiRespList = getWasapiRespList();
PowerMockito.whenNew(WasapiCrawlSelector.class).withArguments(wasapiRespList).thenReturn(mockCrawlSelector);

WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
Mockito.when(mockConn.jsonQuery(anyString())).thenReturn(new WasapiResponse());
Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(wasapiRespList);
WasapiDownloader downloaderSpy = Mockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, args));
Mockito.doReturn(mockConn).when(downloaderSpy).getWasapiConn();

Expand Down

0 comments on commit 7e22eff

Please sign in to comment.