Add configuration for max depth link classifier

VIDA-NYU · Jul 28, 2017 · 61c627d · 61c627d
1 parent 8379674
commit 61c627d
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/src/main/java/focusedCrawler/link/LinkStorageConfig.java b/src/main/java/focusedCrawler/link/LinkStorageConfig.java
@@ -102,6 +102,9 @@ public String getMozKey() {
     @JsonProperty("link_storage.persistent_hashtable.backend")
     private String persistentHashtableBackend = "ROCKSDB";
 
+    @JsonProperty("link_storage.link_classifier.max_depth")
+    private int maxDepth;
+
     public LinkStorageConfig() {
         this.serverConfig = new StorageConfig();
     }
@@ -219,4 +222,8 @@ public DB getPersistentHashtableBackend() {
         }
     }
 
+    public int getMaxDepth() {
+        return maxDepth;
+    }
+
 }
diff --git a/src/main/java/focusedCrawler/link/classifier/LinkClassifierFactory.java b/src/main/java/focusedCrawler/link/classifier/LinkClassifierFactory.java
@@ -2,6 +2,7 @@
 
 import java.nio.file.Paths;
 
+import focusedCrawler.link.LinkStorageConfig;
 import focusedCrawler.util.string.StopList;
 import focusedCrawler.util.string.StopListFile;
 
@@ -14,8 +15,9 @@ public static void setDefaultStoplist(StopList stoplist) {
         LinkClassifierFactory.stoplist = stoplist;
     }
 
-    public static LinkClassifier create(String modelPath, String type) {
-        switch (type) {
+    public static LinkClassifier create(String modelPath, LinkStorageConfig config) {
+        String typeOfClassifier = config.getTypeOfClassifier();
+        switch (typeOfClassifier) {
             case "LinkClassifierBreadthSearch":
                 return new LinkClassifierBreadthSearch();
             case "LinkClassifierBaseline":
@@ -34,9 +36,9 @@ public static LinkClassifier create(String modelPath, String type) {
                 LNClassifier lnClassifier = LNClassifier.create(featureFilePath, modelFilePath, stoplist);
                 return new LinkClassifierImpl(lnClassifier);
             case "MaxDepthLinkClassifier":
-                return new MaxDepthLinkClassifier(1);
+                return new MaxDepthLinkClassifier(config.getMaxDepth());
             default:
-                throw new IllegalArgumentException("Unknown link classifier type: " + type);
+                throw new IllegalArgumentException("Unknown link classifier type: " + typeOfClassifier);
         }
     }
 

diff --git a/src/main/java/focusedCrawler/link/frontier/FrontierManager.java b/src/main/java/focusedCrawler/link/frontier/FrontierManager.java
@@ -93,7 +93,7 @@ public FrontierManager(Frontier frontier, String dataPath, String modelPath,
         this.graphRepository = new BipartiteGraphRepository(dataPath, config.getPersistentHashtableBackend());
         this.hostsManager = new HostManager(Paths.get(dataPath, "data_hosts"), config.getPersistentHashtableBackend());;
         this.schedulerLog = new LogFile(Paths.get(dataPath, "data_monitor", "scheduledlinks.csv"));
-        this.outlinkClassifier = LinkClassifierFactory.create(modelPath, config.getTypeOfClassifier());
+        this.outlinkClassifier = LinkClassifierFactory.create(modelPath, config);
         if (config.getBacklinks()) {
             this.backlinkSurfer = new BacklinkSurfer(config.getBackSurferConfig());
             this.backlinkClassifier = new LinkClassifierHub();