Made metadata cache directory configurable

netarchivesuite · May 21, 2021 · 70bf4d5 · 70bf4d5
1 parent 9205ce7
commit 70bf4d5
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 3 deletions.
diff --git a/common/common-core/src/main/java/dk/netarkivet/common/CommonSettings.java b/common/common-core/src/main/java/dk/netarkivet/common/CommonSettings.java
@@ -259,7 +259,13 @@ public class CommonSettings {
      * The default number of jobs to show in the harvest status section, on one result page.
      */
     public static String HARVEST_STATUS_DFT_PAGE_SIZE = "settings.common.webinterface.harvestStatus.defaultPageSize";
-
+
+    /**
+     * A directory where the webinterface can store cached crawl logs and indexes to minimise the number
+     * of hadoop jobs needed
+     */
+    public static String METADATA_CACHE = "settings.common.webinterface.metadata_cache_dir";
+
     /**
      * <b>settings.common.topLevelDomains.tld</b>: <br>
      * Extra valid top level domain, like .co.uk, .dk, .org., not part of current embedded public_suffix_list.dat file 

diff --git a/harvester/harvester-core/src/main/java/dk/netarkivet/viewerproxy/webinterface/Reporting.java b/harvester/harvester-core/src/main/java/dk/netarkivet/viewerproxy/webinterface/Reporting.java
@@ -169,7 +169,7 @@ public static List<CDXRecord> getMetadataCDXRecordsForJob(long jobid) {
     }
 
     private static File getCDXCacheFile(long jobid) {
-        String cacheDir = "metadata_cache";
+        String cacheDir = Settings.get(CommonSettings.METADATA_CACHE);
         String cdxcache = "cdxcache";
         File cdxdir = new File(new File(cacheDir), cdxcache);
         cdxdir.mkdirs();
@@ -365,7 +365,7 @@ public static File getCrawlLoglinesMatchingRegexp(long jobid, String regexp) {
     }
 
     private static File getCrawlLogCache(long jobid) {
-        String cacheDir = "metadata_cache";
+        String cacheDir = Settings.get(CommonSettings.METADATA_CACHE);
         String crawllog_cache = "crawllog_cache";
         File crawllog_dir = new File(new File(cacheDir), crawllog_cache);
         crawllog_dir.mkdirs();

diff --git a/harvester/harvester-core/src/main/resources/dk/netarkivet/harvester/settings.xml b/harvester/harvester-core/src/main/resources/dk/netarkivet/harvester/settings.xml
@@ -239,6 +239,7 @@ National Library.
         <webinterface>
             <maxCrawlLogInBrowser>1000</maxCrawlLogInBrowser>
             <runningjobsFilteringMethod>database</runningjobsFilteringMethod>
+            <metadata_cache_dir>metadata_cache</metadata_cache_dir>
         </webinterface>
     </harvester>
 </settings>