params rename to crawlId and crawlIdLowerBound (was jobId)

src/edu/stanford/dlss/was/WasapiDownloader.java

@@ -123,10 +123,10 @@ else if ("sha1".equals(algorithm))
 
   private List<Integer> desiredCrawlIds(WasapiCrawlSelector crawlSelector) {
     // TODO: want cleaner grab of int from settings: wasapi-downloader#83
-    Integer myInteger = IntegerValidator.getInstance().validate(settings.jobIdLowerBound());
+    Integer myInteger = IntegerValidator.getInstance().validate(settings.crawlIdLowerBound());
     if (myInteger != null) {
-      int jobsAfter = myInteger.intValue();
-      return crawlSelector.getSelectedCrawlIds(jobsAfter);
+      int crawlsAfter = myInteger.intValue();
+      return crawlSelector.getSelectedCrawlIds(crawlsAfter);
     }
     else
       return crawlSelector.getSelectedCrawlIds(0); // all returns all crawl ids from FileSet
@@ -159,8 +159,8 @@ private List<String> requestParams() {
       params.add("crawl-start-after=" + settings.crawlStartAfter());
     if (settings.crawlStartBefore()!= null)
       params.add("crawl-start-before=" + settings.crawlStartBefore());
-    if (settings.jobId() != null)
-      params.add("crawl=" + settings.jobId());
+    if (settings.crawlId() != null)
+      params.add("crawl=" + settings.crawlId());
     return params;
   }
 

src/edu/stanford/dlss/was/WasapiDownloaderSettings.java

@@ -39,10 +39,10 @@ public class WasapiDownloaderSettings {
   public static final String ACCCOUNT_ID_PARAM_NAME = "accountId";
   public static final String HELP_PARAM_NAME = "help";
   public static final String COLLECTION_ID_PARAM_NAME = "collectionId";
-  public static final String JOB_ID_PARAM_NAME = "jobId";
+  public static final String CRAWL_ID_PARAM_NAME = "crawlId";
   public static final String CRAWL_START_AFTER_PARAM_NAME = "crawlStartAfter";
   public static final String CRAWL_START_BEFORE_PARAM_NAME = "crawlStartBefore";
-  public static final String JOB_ID_LOWER_BOUND_PARAM_NAME = "jobIdLowerBound";
+  public static final String CRAWL_ID_LOWER_BOUND_PARAM_NAME = "crawlIdLowerBound";
   public static final String OUTPUT_BASE_DIR_PARAM_NAME = "outputBaseDir";
   public static final String FILENAME_PARAM_NAME = "filename";
   public static final String CHECKSUM_ALGORITHM_PARAM_NAME = "checksumAlgorithm";
@@ -62,10 +62,10 @@ public class WasapiDownloaderSettings {
     buildArgOption(PASSWORD_PARAM_NAME, "the password for WASAPI server login"),
     buildArgOption(ACCCOUNT_ID_PARAM_NAME, "the ID for the account from which WARC files are downloaded"),
     buildArgOption(COLLECTION_ID_PARAM_NAME, "a collection from which to download crawl files"),
-    buildArgOption(JOB_ID_PARAM_NAME, "a job from which to download crawl files"),
+    buildArgOption(CRAWL_ID_PARAM_NAME, "crawl id from which to download files"),
     buildArgOption(CRAWL_START_AFTER_PARAM_NAME, "only download crawl files created after this date"),
     buildArgOption(CRAWL_START_BEFORE_PARAM_NAME, "only download crawl files created before this date"),
-    buildArgOption(JOB_ID_LOWER_BOUND_PARAM_NAME, "\"last crawl downloaded\": only download crawl files with a higher job ID (not inclusive)"),
+    buildArgOption(CRAWL_ID_LOWER_BOUND_PARAM_NAME, "\"last crawl downloaded\": only download crawl files with a higher crawl ID (not inclusive)"),
     buildArgOption(OUTPUT_BASE_DIR_PARAM_NAME, "destination directory for downloaded WARC files"),
     buildArgOption(FILENAME_PARAM_NAME, "single filename to download"),
     buildArgOption(CHECKSUM_ALGORITHM_PARAM_NAME, "checksum algorithm to use (either md5 or sha1")
@@ -125,8 +125,8 @@ public String collectionId() {
     return settings.getProperty(COLLECTION_ID_PARAM_NAME);
   }
 
-  public String jobId() {
-    return settings.getProperty(JOB_ID_PARAM_NAME);
+  public String crawlId() {
+    return settings.getProperty(CRAWL_ID_PARAM_NAME);
   }
 
   // e.g. 2014-01-01, see https://github.com/WASAPI-Community/data-transfer-apis/tree/master/ait-reference-specification#paths--examples
@@ -139,8 +139,8 @@ public String crawlStartBefore() {
     return settings.getProperty(CRAWL_START_BEFORE_PARAM_NAME);
   }
 
-  public String jobIdLowerBound() {
-    return settings.getProperty(JOB_ID_LOWER_BOUND_PARAM_NAME);
+  public String crawlIdLowerBound() {
+    return settings.getProperty(CRAWL_ID_LOWER_BOUND_PARAM_NAME);
   }
 
   public String outputBaseDir() {
@@ -207,14 +207,14 @@ protected List<String> getSettingsErrorMessages() {
       errMessages.add(ACCCOUNT_ID_PARAM_NAME + " must be an integer (if specified)");
     if (!isNullOrEmpty(collectionId()) && !intValidator.isValid(collectionId()))
       errMessages.add(COLLECTION_ID_PARAM_NAME + " must be an integer (if specified)");
-    if (!isNullOrEmpty(jobId()) && !intValidator.isValid(jobId()))
-      errMessages.add(JOB_ID_PARAM_NAME + " must be an integer (if specified)");
+    if (!isNullOrEmpty(crawlId()) && !intValidator.isValid(crawlId()))
+      errMessages.add(CRAWL_ID_PARAM_NAME + " must be an integer (if specified)");
     if (!isNullOrEmpty(crawlStartBefore()) && !normalizeIso8601Setting(CRAWL_START_BEFORE_PARAM_NAME))
       errMessages.add(CRAWL_START_BEFORE_PARAM_NAME + " must be a valid ISO 8601 date string (if specified)");
     if (!isNullOrEmpty(crawlStartAfter()) && !normalizeIso8601Setting(CRAWL_START_AFTER_PARAM_NAME))
       errMessages.add(CRAWL_START_AFTER_PARAM_NAME + " must be a valid ISO 8601 date string (if specified)");
-    if (!isNullOrEmpty(jobIdLowerBound()) && !intValidator.isValid(jobIdLowerBound()))
-      errMessages.add(JOB_ID_LOWER_BOUND_PARAM_NAME + " must be an integer (if specified)");
+    if (!isNullOrEmpty(crawlIdLowerBound()) && !intValidator.isValid(crawlIdLowerBound()))
+      errMessages.add(CRAWL_ID_LOWER_BOUND_PARAM_NAME + " must be an integer (if specified)");
 
     return errMessages;
   }

test/edu/stanford/dlss/was/TestWasapiDownloaderSettings.java

@@ -24,9 +24,9 @@ public void constructor_readsPropertiesFileAndArgs() throws SettingsLoadExceptio
     // args is a String array, in the style of the `String[] args` param taken by the main method of a Java class.
     // JVM splits the whole command line argument string on whitespace, and passes the resultant String array into main, so
     // the below args array would come from something like:
-    //   wasapi-downloader -h --collectionId 123 --jobId=456 --crawlStartAfter 2014-03-14 --crawlStartBefore=2017-03-14
+    //   wasapi-downloader -h --collectionId 123 --crawlId=456 --crawlStartAfter 2014-03-14 --crawlStartBefore=2017-03-14
     // The Apache CLI parser should be able to handle all of those different styles of argument without any trouble.
-    String[] args = { "-h", "--collectionId", "123", "--jobId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14" };
+    String[] args = { "-h", "--collectionId", "123", "--crawlId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14" };
     WasapiDownloaderSettings settings = new WasapiDownloaderSettings(WasapiDownloader.SETTINGS_FILE_LOCATION, args);
 
     assertEquals("baseurl value should have come from settings file", settings.baseUrlString(), "https://example.org/");
@@ -37,7 +37,7 @@ public void constructor_readsPropertiesFileAndArgs() throws SettingsLoadExceptio
     assertEquals("outputBaseDir value should have come from settings file", settings.outputBaseDir(), "test/outputBaseDir/");
     assertEquals("checksumAlgorithm value should have come from settings file", settings.checksumAlgorithm(), "md5");
     assertEquals("collectionId value should have come from args", settings.collectionId(), "123");
-    assertEquals("jobId value should have come from args", settings.jobId(), "456");
+    assertEquals("crawlId value should have come from args", settings.crawlId(), "456");
     assertEquals("crawlStartAfter value should have come from args", settings.crawlStartAfter(), "2014-03-14");
     assertEquals("crawlStartBefore value should have come from args", settings.crawlStartBefore(), "2017-03-14");
     assertTrue("shouldDisplayHelp value should have come from args", settings.shouldDisplayHelp());
@@ -46,9 +46,9 @@ public void constructor_readsPropertiesFileAndArgs() throws SettingsLoadExceptio
   @Test
   @SuppressWarnings({"checkstyle:NoWhitespaceAfter", "checkstyle:MethodLength"})
   public void getHelpAndSettingsMessage_containsUsageAndSettingsInfo() throws SettingsLoadException {
-    //TODO: if settings validation flags possibly nonsensical/redundant combos like jobId
-    // and jobIdLowerBound, this test might have to be broken up a bit.
-    String[] args = { "-h", "--collectionId", "123", "--jobId=456", "--jobIdLowerBound=400",
+    //TODO: if settings validation flags possibly nonsensical/redundant combos like crawlId and crawlIdLowerBound,
+    // then this test might have to be broken up a bit.
+    String[] args = { "-h", "--collectionId", "123", "--crawlId=456", "--crawlIdLowerBound=400",
         "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14", "--filename=filename.warc.gz" };
     WasapiDownloaderSettings settings = new WasapiDownloaderSettings(WasapiDownloader.SETTINGS_FILE_LOCATION, args);
 
@@ -64,17 +64,17 @@ public void getHelpAndSettingsMessage_containsUsageAndSettingsInfo() throws Sett
     assertThat("helpAndSettingsMsg lists crawlStartAfter arg", helpAndSettingsMsg, containsString("--crawlStartAfter <arg>"));
     assertThat("helpAndSettingsMsg lists crawlStartBefore arg", helpAndSettingsMsg, containsString("--crawlStartBefore <arg>"));
     assertThat("helpAndSettingsMsg lists help flag", helpAndSettingsMsg, containsString("-h,--help"));
-    assertThat("helpAndSettingsMsg lists jobId arg", helpAndSettingsMsg, containsString("--jobId <arg>"));
-    assertThat("helpAndSettingsMsg lists jobIdLowerBound arg", helpAndSettingsMsg, containsString("--jobIdLowerBound <arg>"));
+    assertThat("helpAndSettingsMsg lists crawlId arg", helpAndSettingsMsg, containsString("--crawlId <arg>"));
+    assertThat("helpAndSettingsMsg lists crawlIdLowerBound arg", helpAndSettingsMsg, containsString("--crawlIdLowerBound <arg>"));
     assertThat("helpAndSettingsMsg lists filename arg", helpAndSettingsMsg, containsString("--filename <arg>"));
     assertThat("helpAndSettingsMsg lists checksumAlgorithm arg", helpAndSettingsMsg, containsString("--checksumAlgorithm <arg>"));
 
     assertThat("helpAndSettingsMsg hides password value", helpAndSettingsMsg, containsString("password : [password hidden]"));
     assertThat("helpAndSettingsMsg lists crawlStartAfter value", helpAndSettingsMsg, containsString("crawlStartAfter : 2014-03-14"));
     assertThat("helpAndSettingsMsg lists crawlStartBefore value", helpAndSettingsMsg, containsString("crawlStartBefore : 2017-03-14"));
-    assertThat("helpAndSettingsMsg lists jobIdLowerBound value", helpAndSettingsMsg, containsString("jobIdLowerBound : 400"));
+    assertThat("helpAndSettingsMsg lists crawlIdLowerBound value", helpAndSettingsMsg, containsString("crawlIdLowerBound : 400"));
     assertThat("helpAndSettingsMsg lists help flag value", helpAndSettingsMsg, containsString("help : true"));
-    assertThat("helpAndSettingsMsg lists jobId value", helpAndSettingsMsg, containsString("jobId : 456"));
+    assertThat("helpAndSettingsMsg lists crawlId value", helpAndSettingsMsg, containsString("crawlId : 456"));
     assertThat("helpAndSettingsMsg lists collectionId value", helpAndSettingsMsg, containsString("collectionId : 123"));
     assertThat("helpAndSettingsMsg lists baseurl value", helpAndSettingsMsg, containsString("baseurl : https://example.org"));
     assertThat("helpAndSettingsMsg lists authurl value", helpAndSettingsMsg, containsString("authurl : https://example.org/login"));
@@ -96,7 +96,7 @@ public void argsOverrideSettings() throws SettingsLoadException {
   @Test
   @SuppressWarnings("checkstyle:NoWhitespaceAfter")
   public void toString_aliasesGetHelpAndSettingsMessage() throws SettingsLoadException {
-    String[] args = { "-h", "--collectionId", "123", "--jobId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14" };
+    String[] args = { "-h", "--collectionId", "123", "--crawlId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14" };
     WasapiDownloaderSettings settings = new WasapiDownloaderSettings(WasapiDownloader.SETTINGS_FILE_LOCATION, args);
 
     assertSame("toString should return the same String obj as getHelpAndSettingsMessage", settings.getHelpAndSettingsMessage(), settings.toString());
@@ -150,8 +150,8 @@ public void getSettingsErrorMessages_listsAllErrors() {
     internalSettings.setProperty(WasapiDownloaderSettings.ACCCOUNT_ID_PARAM_NAME, "z26");
     internalSettings.setProperty(WasapiDownloaderSettings.OUTPUT_BASE_DIR_PARAM_NAME, "does/not/exist");
     internalSettings.setProperty(WasapiDownloaderSettings.COLLECTION_ID_PARAM_NAME, "a1");
-    internalSettings.setProperty(WasapiDownloaderSettings.JOB_ID_PARAM_NAME, "b2");
-    internalSettings.setProperty(WasapiDownloaderSettings.JOB_ID_LOWER_BOUND_PARAM_NAME, "c3");
+    internalSettings.setProperty(WasapiDownloaderSettings.CRAWL_ID_PARAM_NAME, "b2");
+    internalSettings.setProperty(WasapiDownloaderSettings.CRAWL_ID_LOWER_BOUND_PARAM_NAME, "c3");
     internalSettings.setProperty(WasapiDownloaderSettings.CRAWL_START_BEFORE_PARAM_NAME, "01/01/2001");
     internalSettings.setProperty(WasapiDownloaderSettings.CRAWL_START_AFTER_PARAM_NAME, "12/31/2010");
     internalSettings.setProperty(WasapiDownloaderSettings.CHECKSUM_ALGORITHM_PARAM_NAME, "foo");
@@ -164,10 +164,10 @@ public void getSettingsErrorMessages_listsAllErrors() {
     assertThat("error messages has entry for invalid account ID", errMsgs, hasItem("accountId must be an integer (if specified)"));
     assertThat("error messages has entry for invalid outputBaseDir", errMsgs, hasItem("outputBaseDir is required (and must be an extant, writable directory)"));
     assertThat("error messages has entry for invalid collectionId", errMsgs, hasItem("collectionId must be an integer (if specified)"));
-    assertThat("error messages has entry for invalid jobId", errMsgs, hasItem("jobId must be an integer (if specified)"));
+    assertThat("error messages has entry for invalid crawlId", errMsgs, hasItem("crawlId must be an integer (if specified)"));
     assertThat("error messages has entry for invalid crawlStartBefore", errMsgs, hasItem("crawlStartBefore must be a valid ISO 8601 date string (if specified)"));
     assertThat("error messages has entry for invalid crawlStartAfter", errMsgs, hasItem("crawlStartAfter must be a valid ISO 8601 date string (if specified)"));
-    assertThat("error messages has entry for invalid jobIdLowerBound", errMsgs, hasItem("jobIdLowerBound must be an integer (if specified)"));
+    assertThat("error messages has entry for invalid crawlIdLowerBound", errMsgs, hasItem("crawlIdLowerBound must be an integer (if specified)"));
     assertThat("error messages has entry for invalid checksumAlgorithm", errMsgs, hasItem("checksumAlgorithm is required and must be md5 or sha1"));
   }
 

test/edu/stanford/dlss/was/TestWasapiDownloader_PowerMock.java

@@ -38,7 +38,8 @@ public void main_callsExecutFromCmdLine() throws Exception {
 
   @Test
   public void main_executesFileSetRequest_usesAllAppropArgsSettings() throws Exception {
-    String[] args = {"--collectionId", "123", "--jobId=456", "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14", "--username=Fred" };
+    String[] args = {"--collectionId", "123", "--crawlId=456",
+        "--crawlStartAfter", "2014-03-14", "--crawlStartBefore=2017-03-14", "--username=Fred" };
     WasapiConnection mockConn = Mockito.mock(WasapiConnection.class);
     Mockito.when(mockConn.pagedJsonQuery(anyString())).thenReturn(null);
     WasapiDownloader downloaderSpy = PowerMockito.spy(new WasapiDownloader(WasapiDownloader.SETTINGS_FILE_LOCATION, args));
@@ -144,9 +145,9 @@ public void downloadSelectedWarcs_callsDownloadAndValidateFile() throws Exceptio
 
   @Test
   @SuppressWarnings("checkstyle:NoWhitespaceAfter")
-  public void downloadSelectedWarcs_byJobIdLowerBound() throws Exception {
+  public void downloadSelectedWarcs_byCrawlIdLowerBound() throws Exception {
     String argValue = "666";
-    String[] args = { "--jobIdLowerBound=" + argValue };
+    String[] args = { "--crawlIdLowerBound=" + argValue };
 
     WasapiCrawlSelector mockCrawlSelector = PowerMockito.mock(WasapiCrawlSelector.class);
     List<WasapiResponse> wasapiRespList = getWasapiRespList();