update

Author: bob7783

nlp_class/sentiment.py

@@ -34,22 +34,13 @@
 
 # load the reviews
 # data courtesy of http://www.cs.jhu.edu/~mdredze/datasets/sentiment/index2.html
-positive_reviews = BeautifulSoup(open('electronics/positive.review').read())
+positive_reviews = BeautifulSoup(open('electronics/positive.review').read(), features="html5lib")
 positive_reviews = positive_reviews.findAll('review_text')
 
-negative_reviews = BeautifulSoup(open('electronics/negative.review').read())
+negative_reviews = BeautifulSoup(open('electronics/negative.review').read(), features="html5lib")
 negative_reviews = negative_reviews.findAll('review_text')
 
-# there are more positive reviews than negative reviews
-# so let's take a random sample so we have balanced classes
-# np.random.shuffle(positive_reviews)
-# positive_reviews = positive_reviews[:len(negative_reviews)]
 
-# we can also oversample the negative reviews
-diff = len(positive_reviews) - len(negative_reviews)
-idxs = np.random.choice(len(negative_reviews), size=diff)
-extra = [negative_reviews[i] for i in idxs]
-negative_reviews += extra
 
 # first let's just try to tokenize the text using nltk's tokenizer
 # let's take the first review for example: