oversample

bob7783 · bob7783 · commit 232f10e7a4da · 2018-04-17T15:19:19.000-04:00
diff --git a/nlp_class/sentiment.py b/nlp_class/sentiment.py
@@ -41,8 +41,14 @@
 
 # there are more positive reviews than negative reviews
 # so let's take a random sample so we have balanced classes
-np.random.shuffle(positive_reviews)
-positive_reviews = positive_reviews[:len(negative_reviews)]
+# np.random.shuffle(positive_reviews)
+# positive_reviews = positive_reviews[:len(negative_reviews)]
+
+# we can also oversample the negative reviews
+diff = len(positive_reviews) - len(negative_reviews)
+idxs = np.random.choice(len(negative_reviews), size=diff)
+extra = [negative_reviews[i] for i in idxs]
+negative_reviews += extra
 
 # first let's just try to tokenize the text using nltk's tokenizer
 # let's take the first review for example:
