Modified FileResolver to return empty if http response code is not 200.

netarchivesuite · Jan 11, 2022 · cbc5199 · cbc5199
1 parent 34952e9
commit cbc5199
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/utils/hadoop/HadoopJob.java b/common/common-core/src/main/java/dk/netarkivet/common/utils/hadoop/HadoopJob.java
@@ -81,6 +81,9 @@ public void prepareJobInputOutput(FileSystem fileSystem) {
         fileCount = filePaths.size();
         log.info("{} found {} file(s) matching pattern '{}' to add to input file for {} job {}",
                 fileResolver.getClass().getName(), fileCount, filenamePattern, jobType, jobID);
+        if (fileCount == 0) {
+            log.warn("Zero input files found for job {}, {}. Proceeding with caution.", jobType, jobID);
+        }
         try {
             HadoopJobUtils.writeHadoopInputFileLinesToInputFile(filePaths, localInputTempFile);
         } catch (IOException e) {

diff --git a/.../common-core/src/main/java/dk/netarkivet/common/utils/service/FileResolverRESTClient.java b/.../common-core/src/main/java/dk/netarkivet/common/utils/service/FileResolverRESTClient.java
@@ -74,6 +74,11 @@ private List<Path> getPaths(Pattern filepattern, boolean exactfilename) {
             CloseableHttpClient httpClient = clientBuilder.getHttpsClient();
 
             try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
+                int statusCode = httpResponse.getStatusLine().getStatusCode();
+                if (statusCode != 200) {
+                    log.warn("FileResolver call to {} returned status code {}.", uri, statusCode);
+                    return new ArrayList<>();
+                }
                 InputStream istr = httpResponse.getEntity().getContent();
                 List<String> results = IOUtils.readLines(istr);
                 return results.stream()