Updated global lexica.

clir · Apr 23, 2015 · b5d4ceb · b5d4ceb
1 parent 401d866
commit b5d4ceb
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 41 deletions.
diff --git a/src/main/java/edu/emory/clir/clearnlp/component/AbstractStatisticalComponent.java b/src/main/java/edu/emory/clir/clearnlp/component/AbstractStatisticalComponent.java
@@ -31,6 +31,8 @@
 
 import edu.emory.clir.clearnlp.classification.instance.StringInstance;
 import edu.emory.clir.clearnlp.classification.model.StringModel;
+import edu.emory.clir.clearnlp.classification.trainer.AbstractOnlineTrainer;
+import edu.emory.clir.clearnlp.classification.trainer.AdaGradSVM;
 import edu.emory.clir.clearnlp.classification.vector.StringFeatureVector;
 import edu.emory.clir.clearnlp.component.configuration.AbstractConfiguration;
 import edu.emory.clir.clearnlp.component.evaluation.AbstractEval;
@@ -358,32 +360,32 @@ public boolean isDecodeOrEvaluate()
 
 	protected void onlineTrainSingleAdaGrad(List<DEPTree> trees)
 	{
-//		double currScore = onlineScore(trees);
-//		if (currScore == 100) return;
-//		onlineBootstrap(trees);
-//		
-//		AbstractOnlineTrainer trainer = new AdaGradSVM(s_models[0], 0, 0, false, 0.01, 0.1, 0d);
-//		byte[] prevModels;
-//		double prevScore;
-//		
-//		try
-//		{
-//			while (true)
-//			{
-//				prevModels = toByteArray();
-//				prevScore  = currScore;
-//				
-//				trainer.train();
-//				currScore = onlineScore(trees);
-//				
-//				if (prevScore >= currScore)
-//				{
-//					initDecode(prevModels);
-//					break;
-//				}
-//			}			
-//		}
-//		catch (Exception e) {e.printStackTrace();}
+		double currScore = onlineScore(trees);
+		if (currScore == 100) return;
+		onlineBootstrap(trees);
+
+		AbstractOnlineTrainer trainer = new AdaGradSVM(s_models[0], 0, 0, false, 0.01, 0.1, 0d);
+		byte[] prevModels;
+		double prevScore;
+
+		try
+		{
+			while (true)
+			{
+				prevModels = toByteArray();
+				prevScore  = currScore;
+
+				trainer.train();
+				currScore = onlineScore(trees);
+
+				if (prevScore >= currScore)
+				{
+					initDecode(prevModels);
+					break;
+				}
+			}			
+		}
+		catch (Exception e) {e.printStackTrace();}
 	}
 
 	protected double onlineScore(List<DEPTree> trees)

diff --git a/src/main/java/edu/emory/clir/clearnlp/component/utils/GlobalLexica.java b/src/main/java/edu/emory/clir/clearnlp/component/utils/GlobalLexica.java
@@ -23,8 +23,6 @@
 
 import org.w3c.dom.Element;
 
-import edu.emory.clir.clearnlp.collection.tree.PrefixTree;
-import edu.emory.clir.clearnlp.ner.NERInfoSet;
 import edu.emory.clir.clearnlp.util.DSUtils;
 import edu.emory.clir.clearnlp.util.XmlUtils;
 
@@ -35,7 +33,6 @@
 public class GlobalLexica
 {
 	static private List<Map<String,Set<String>>> distributional_semantics_words;
-	static private PrefixTree<String,NERInfoSet> ner_dictionary;
 
 	static public void init(InputStream in)
 	{
@@ -44,8 +41,6 @@ static public void init(InputStream in)
 		if (eLexica == null) return;
 
 		initDistributionalSemanticsWords(XmlUtils.getTrimmedTextContents(eLexica, "distributional_semantics_path"));
-		Element e = XmlUtils.getFirstElementByTagName(eLexica, "ner_dictionary_path");
-		if (e != null) initNamedEntityDictionary(XmlUtils.getTrimmedTextContent(e));
 	}
 
 	static public void initDistributionalSemanticsWords(List<String> paths)
@@ -62,14 +57,4 @@ static public String[] getDistributionalSemanticFeatures(int index, String word)
 		set.toArray(t);
 		return t;
 	}
-
-	static public void initNamedEntityDictionary(String path)
-	{
-		ner_dictionary = NLPUtils.getNERDictionary(path);
-	}
-
-	static public PrefixTree<String,NERInfoSet> getNamedEntityDictionary()
-	{
-		return ner_dictionary;
-	}
 }
diff --git a/src/main/java/edu/emory/clir/clearnlp/util/DSUtils.java b/src/main/java/edu/emory/clir/clearnlp/util/DSUtils.java
@@ -457,4 +457,11 @@ static public <T>Set<T> toHashSet(T... items)
 		for (T item : items) set.add(item);
 		return set;
 	}
+
+	static public String[] toArray(Collection<String> col)
+	{
+		String[] array = new String[col.size()];
+		col.toArray(array);
+		return array;
+	}
 }