fixed random seed and perfect score calculation

FakeNewsChallenge · Mar 2, 2017 · 4678604 · 4678604
1 parent 3af02ad
commit 4678604
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 12 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "fnc-1"]
 	path = fnc-1
-	url = git@github.com:FakeNewsChallenge/fnc-1.git
+	url = https://github.com/FakeNewsChallenge/fnc-1
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ Credit:
 * HJ van Veen (GitHub/Slack: @mlwave)
 * Delip Rao (GitHub: @delip, Slack: @dr)
 * James Thorne (GitHub/Slack: @j6mes)
-
+* Yuxi Pan (GitHub: @yuxip, Slack: @yuxipan)
 
 ## Questions / Issues
 Please raise questions in the slack group [fakenewschallenge.slack.com](https://fakenewschallenge.slack.com)
@@ -72,11 +72,17 @@ The ``report_score`` function in ``utils/score.py`` is based off the original sc
 
 This will print a confusion matrix and a final score your classifier. We provide the scores for a classifier with a simple set of features which you should be able to match and eventually beat!
 
-|           	| agree 	| disagree 	| discuss 	| unrelated 	|
-|-----------	|-------	|----------	|---------	|-----------	|
-| agree     	| 106   	| 13      	| 56     	| 0         	|
-| disagree  	| 0     	| 3        	| 1      	| 0            	|
-| discuss   	| 553    	| 108      	| 1460     	| 150       	|
-| unrelated 	| 116   	| 29       	| 268    	| 6885      	|
+-------------------------------------------------------------
+|           |   agree   | disagree  |  discuss  | unrelated |
+-------------------------------------------------------------
+|   agree   |    94     |    23     |    58     |     7     |
+-------------------------------------------------------------
+| disagree  |     3     |     1     |     2     |     0     |
+-------------------------------------------------------------
+|  discuss  |    624    |    121    |   1341    |    106    |
+-------------------------------------------------------------
+| unrelated |    88     |    24     |    237    |   7400    |
+-------------------------------------------------------------
+
 
-Score: 3473.0 out of 4277.5     (81.19%)
+Score: 3493.75 out of 4317.25     (80.92%)
diff --git a/fnc_kfold.py b/fnc_kfold.py
@@ -64,7 +64,7 @@ def generate_features(stances,dataset,name):
         actual = [LABELS[int(a)] for a in y_test]
 
         fold_score, _ = score_submission(actual, predicted)
-        max_fold_score, _ = score_submission(predicted, predicted)
+        max_fold_score, _ = score_submission(actual, actual)
 
         score = fold_score/max_fold_score
 

diff --git a/utils/generate_test_splits.py b/utils/generate_test_splits.py
@@ -8,7 +8,7 @@ def generate_hold_out_split (dataset, training = 0.8, base_dir="splits"):
     r.seed(1489215)
 
     article_ids = list(dataset.articles.keys())  # get a list of article ids
-    random.shuffle(article_ids)  # and shuffle that list
+    r.shuffle(article_ids)  # and shuffle that list
 
 
     training_ids = article_ids[:int(training * len(article_ids))]
@@ -59,4 +59,4 @@ def get_stances_for_folds(dataset,folds,hold_out):
                     stances_folds[fold_id].append(stance)
                 fold_id += 1
 
-    return stances_folds,stances_hold_out
+    return stances_folds,stances_hold_out