From 9dda59dc2c25d0ad5d93d8c9095f1bdb9c964ee1 Mon Sep 17 00:00:00 2001 From: Victor Neo Date: Mon, 19 Mar 2012 20:11:32 +0800 Subject: [PATCH] Initial commit --- LICENSE | 19 +++++++++++ README.md | 51 ++++++++++++++++++++++++++++ classification.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++ happy.txt | 80 ++++++++++++++++++++++++++++++++++++++++++++ happy_test.txt | 10 ++++++ sad.txt | 80 ++++++++++++++++++++++++++++++++++++++++++++ sad_test.txt | 10 ++++++ 7 files changed, 334 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 classification.py create mode 100644 happy.txt create mode 100644 happy_test.txt create mode 100644 sad.txt create mode 100644 sad_test.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3ae467a --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012 Victor Neo + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a3fdbb --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +Twitter Sentimental Analysis +========= + +This tool is an implementation of the [Twitter Sentiment analysis tool][1] +described on [Laurent Luce's blog][2]. + +Requirements +------------ +* [NLTK][3] and its dependencies + +You may install NLTK by using pip: + + pip install nltk + + +How to use +---------- + +1. Clone this project +2. Install NLTK (if it is not installed) +3. Run the classifier + + git clone git://github.com/victorneo/Twitter-Sentimental-Analysis.git + pip install nltk + python classification.py + + +Training data +------------- + +The training data is obtained from the Twitter Search API with the keywords +`I am happy` and `I am sad` for happy (positive) and sad (negative) tweets. +There is a total of 160 tweets used for training (80 / 80 distribution). + +To add more training data, add in new _happy_ tweets to `happy.txt` and _sad_ tweets +to `sad.txt` using one line for each new tweet. + + +Test data +--------- + +Test data are separated into `happy_test.txt` and `sad_test.txt`. A total of +20 tweets are used for test (10 / 10 distribution). + +To add more test data, add in new _happy_ tweets to `happy_test.txt` and _sad_ +tweets to `sad_test.txt` using one line for each new tweet. + + +[1]: http://www.laurentluce.com/posts/twitter-sentiment-analysis-using-python-and-nltk/ +[2]: http://www.laurentluce.com/ +[3]: http://www.nltk.org/ diff --git a/classification.py b/classification.py new file mode 100644 index 0000000..f0daf03 --- /dev/null +++ b/classification.py @@ -0,0 +1,84 @@ +''' +An implementation of the Twitter analysis tool mentioned on +http://www.laurentluce.com/posts/twitter-sentiment-analysis-using-python-and-nltk/ + +happy.txt contains 80 tweets that are happy (positive), +sad.txt contains 80 tweets that are sad (negative). + +happy_test.txt and sad_test.txt contains 10 positive +and 10 negative tweets respectively to test the +classifier. +''' +import nltk +from nltk.classify.naivebayes import NaiveBayesClassifier + + +def get_words_in_tweets(tweets): + all_words = [] + for (words, sentiment) in tweets: + all_words.extend(words) + return all_words + + +def get_word_features(wordlist): + wordlist = nltk.FreqDist(wordlist) + word_features = wordlist.keys() + return word_features + + +def read_tweets(fname, t_type): + tweets = [] + f = open(fname, 'r') + line = f.readline() + while line != '': + tweets.append([line, t_type]) + line = f.readline() + f.close() + return tweets + + +def extract_features(document): + document_words = set(document) + features = {} + for word in word_features: + features['contains(%s)' % word] = (word in document_words) + return features + + +def classify_tweet(tweet): + return \ + classifier.classify(extract_features(nltk.word_tokenize(tweet))) + + +# read in postive and negative training tweets +pos_tweets = read_tweets('happy.txt', 'positive') +neg_tweets = read_tweets('sad.txt', 'negative') + +# filter away words that are less than 3 letters to form the training data +tweets = [] +for (words, sentiment) in pos_tweets + neg_tweets: + words_filtered = [e.lower() for e in words.split() if len(e) >= 3] + tweets.append((words_filtered, sentiment)) + + +# extract the word features out from the training data +word_features = get_word_features(\ + get_words_in_tweets(tweets)) + + +# get the training set and train the Naive Bayes Classifier +training_set = nltk.classify.util.apply_features(extract_features, tweets) +classifier = NaiveBayesClassifier.train(training_set) + + +# read in the test tweets and check accuracy +# to add your own test tweets, add them in the respective files +test_tweets = read_tweets('happy_test.txt', 'positive') +test_tweets.extend(read_tweets('sad_test.txt', 'negative')) +total = accuracy = float(len(test_tweets)) + +for tweet in test_tweets: + if classify_tweet(tweet[0]) != tweet[1]: + accuracy -= 1 + +print('Total accuracy: %f%% (%d/20).' % (accuracy / total * 100, accuracy)) diff --git a/happy.txt b/happy.txt new file mode 100644 index 0000000..202e447 --- /dev/null +++ b/happy.txt @@ -0,0 +1,80 @@ +I am LOVIN my Life right about now! I'm loving the people God is placing in my life. #Happy&Focused! Striving to be the BEST WOMAN I can be! +Happy St.Patrick's Day! Wasn't always easy to celebrate it +no one understands how happy i am right now. (': +AGH. DELETE DELETE DELETE. I JUST MEANT HAPPY. I AM NOT A STRIPPER. OH, GAWD. +she feels as happy as i am. and i am happy to say that we r both blessed. been the best spring break ever, +I am so happy, it's kind of unbelievable. +Ignorance is bliss and i need that shyt...I am happy. Thats just the saddest lie. #Cudi +WMYB on the radio and i am one happy person(: #thinkingpositive +realize you fucking twat. I'm not dragging in a sad way. I am actually quite happy! And I wouldn't say I don't care for you. +because of you i am happy +i am very happy today ........yuhuuuuuuuiiiiiiii +I am soooo happy I got a chance to hang my boo @BlaqBetty2 last night!!! Its been way too long!!! Muah! +I am so happy for you and your drivers. That was an amazing finish for the MRW teams today +17 more days until Opening Day. To say that I am happy about this is an understatement. The #mlb2012 season will be the BEST one ever! @mlb +I am happy for the friendships I am developing with some people on twisters +" Convert Kudankulam plant into a gasoline-based one" says Wise-Admiral Ramdas. I am happy he is a 'former' Naval Chief now. +Today was a perfect day with my raza I am happy +i'm assuming it's your bday where you're at even if it's not where i am lol . Well HAPPY BIRTHDAY ! +I don't specifically know why I like this picture, but I do. I am happy. :) +Ignorance is bliss and i need that shyt...I am happy. Thats just the saddest lie. #Cudi +Happy Sunday. I went with blue eyes today. http://t.co/x4Tqbs0y +i am happy with jointwitte +I'm am so happy the PTSD,ADHD, psycho is now following me. Does he still have a HIGHER SECURITY CLEARENCE than POTUS OBAMA? +i'm assuming it's your bday where you're at even if it's not where i am lol . Well HAPPY BIRTHDAY ! +Dear #hiltongardeninn I am NOT happy with the fact that I got stuck with the room beside the ice machine. #goldmember +I am so happy with my life! +i'm pretty happy with where i am now. +Happy Birthday Ton! I am still working hard at what we both started, getting our music out there. RIP Family http://t.co/jURzc6WH +Jamie, your daughter is stunning. I am so happy for you girl. +As happy as I am to see my frends I wish it was summer(: +I am happy you're sober. :) +I am brewing w envy. Ok fine. Happy for you. :/ +I still check on you from time to time. i'm glad you're happy. i really am. +Jamie, your daughter is stunning. I am so happy for you girl. +I am so happy, it's kind of unbelievable. +:'D YUUUUSSSS lmao yall don't know how happy i am #sj3 +I said I will close the distance up. I did it, so glad to have you around. Happy Birthday baby ❤ http://t.co/919VOI5v +Happy 16th baby girl ❤ I love you!!! @khamiltonnn http://t.co/axejLg1X +I am so happy I am your friend right now. +*giggles and smiles brightly* I am glad we are both happy too. Life is good. +I'm happy for him...really, I am. She's an amazing girl, and they deserve each other. He's happy & thats all that matters...right?..... +Dearest one can't be replace, even you're annoys me. But you're the one I love MOST! Happy B'day Brenda http://t.co/SXFW7EwM +This has been a good weekend! I am so happy that I am back with everyone... now for the busiest week of my life +If you don't tweet for a while you can do 3 in a row! I am super duper happy to be home with my sweetie! He is the best! I love him! +We finished our first season of @TheBEATDance & I am so happy & proud & thankful & overwhelmed & lots of other good stuff! So Amazing #2013 +I am so happy because I�ve keep a $100,00 Starbucks GIFT CARD 4 Free - I get it here: http://t.co/en6kV48W +Did I mention that I am sad I have to go back to work tomorrow....but happy I can just do my job for now on..... +I am happy, I am successful, bein in my own living space, bein in the masters prog I chose for myself, a life full of positivity...yay me! +I Am So HAPPY . +Looking through old pics and realizing everything happens for a reason. So happy with where I am right now +Disney doesn't have TLC l, therefore this can not be the happiest place on earth because I am not happy... +I wish for a stupid dream for the rest of the week. Maybe a dream where I am President, no skip that. *paging* Happy Monday Twitter.☺☺ +I am sorry! Happy birthdax :) +had a taste for Jims...& I just had the nastiest steak hoagy from @JimsSteakout...very disappointed, a waste of my $11..i am not happy! +im so happy we have school tmrw because that means one day closer to summer?!(: am i right +17 more days until Opening Day. To say that I am happy about this is an understatement. The #mlb2012 season will be the BEST one ever! @mlb +haha, i am def happy about tanning lol. it is such a stress reliever. +Oo I think no licking my lolly sounds good. I am happy to hear you are getting a little better!! +patamon : yayyy!! I am so happy, now I can play games with you!! pata~ +As long as she's happy then so am I.. My life <3 http://t.co/ZqqqbEih +I'm always happy! Photo by @krisstephens http://t.co/3T4VGuDb • you look really hot babe xxx < Look At! +Nothing's wrong with thay @TheRealTank: I'm always happy! Photo by http://t.co/ywiJKEyD +Sorry I post so many screen shots, I just like showing how happy I am(: #sorrynotsorry #sorryboutit +Give it up give it up give it up for SHINee .. Thank you .. I am happy because of you, because of sherlock +When I am happy my heart starts to slow +My little brother is re evaluating my life I'm back to a B+ Haha... # @SavinTheWorldKP kirsti must be how happy I am from last night +My hair gone bye bye and I am so happy to say I am glad to have my hair back. (Even though this is a hard thing to see) +My life has no purpose, no direction, no aim, no meaning, and yet I’m happy. I can’t figure it out. What am I doing right? +i am X-5’s fan & now ，i am NUEST fan too。。。i am from China！！！happy to follow！！！！ +I'm always happy! Photo by @krisstephens http://t.co/Yy5WTHGn • you look really hot babe xxx +I am happy, I am successful, bein in my own living space, bein in the masters prog I chose for myself, a life full of positivity...yay me! +Alli, thanks for favoriteing my tweet! I like literally screamed! I am so happy Please follow me!!!?:) +After several mini mental breakdowns 2day I am happy 2 say tht evrythng is starting 2 fall n2 place. Things happen 4 a reason. #justgowithit +Why am I so happy right now!? +Why the fuck am I so awake and so happy? +just few hours to have you in America again!!! #TWonUS cant describe how happy I am :D +Happy for Big 12 that KU won but I am a huge Robbie Hummel fan! Great player who really is what college hoops is suppose to be about! +I am happy for who I am because I went through so much shit to become the person who I am today. +No school in the am I'm soo happy ! +I am happy and all that I'm legal now but I would do anything to be like 5 again #foreveryoung diff --git a/happy_test.txt b/happy_test.txt new file mode 100644 index 0000000..8a2bda4 --- /dev/null +++ b/happy_test.txt @@ -0,0 +1,10 @@ +I'm happy for him...really, I am. She's an amazing girl, and they deserve each other. He's happy & thats all that matters...right?..... +Feel so happy with no reason... Just happy... Hey my brain, am I missing something? :)) +We finished our first season of @TheBEATDance & I am so happy & proud & thankful & overwhelmed & lots of other good stuff! So Amazing #2013 +am i allowed to be happy about something, or do yo wanna distroy the little i have left? +I am so happy right now I can't even focus on anything else +Why am I being sneaked around her fam when I'm open about us.... But we both happy shit don't add up. +Heavens suppose to be the happiest place in the world I am happy everyday with the people I love but I feel like I live in heaven everyday:) +I am so happy since I have get an $100,00 STARBUCKS GIFT-CARD for Free. I grab it here http://t.co/cg8M1Ubq +I am one #happy girl :) +I Am So HAPPY . diff --git a/sad.txt b/sad.txt new file mode 100644 index 0000000..7712575 --- /dev/null +++ b/sad.txt @@ -0,0 +1,80 @@ +When I think about #TrayvonMartin I can not help but worry about my own son! I am so sad and angry right now! Have been all day actually. +#iLikePeopleWho makes me laugh when I am sad. +Sometimes I just instantly get sad when I realize how blessed I am and I think about homeless people sleeping outside and hungry :( +I have zero pics from last night very sad :( but hey I am proud of that hat!! +I hear ya. I tried for a while and am now watching Goldmember. So sad +I feel happy, I feel sad. And I don't know who I am.,It’s not the way you plan it is how you make it happend. +it was so good! But so sad. I cried like a baby!! I am so sad that its ending :( +no I am not in NYC these days... I know I am sad as well +Why am I suddenly all sad? +follows everyone but everyone doesn't follow back now i am sad +Reading past convos i had with females, i am so oblivious and CANNOT take a hint. This is sad +I am sad that the chief of police has already made up his mind that the shooter was protecting himself. +I am on my PC! The link is not working. :: sad:: +As mad and sad as I am about Mike being killed, it was such a Brilliant example of tragic writing at its best. +ohhhh come on ! now im sad. I am the one with really no friends at all. +Oh my good!!!! Too much feelings inside and any way to show them... How sad girl am i?? A big one +meh i am sad +Recently found out blue mountain state is over . . . What am I suppose to watch now #sad +seeing you at the same place i am at makes me very sad. sad that you didnt even say ONE FUCKING word to me. +I am actually kind of sad that its not flannel weather anymore... #Iloveflannel #bestmaterialever #afterwoolandleather +if any of your guys are followed by @diggy_simmons your rlly lucky cus im like trying and i need help cus i am going crazy. ugh i am sad :( +i am kinda sad. lol +Whenever I am sad, angry, or bored, I eat. +Good night all. I am so sad that Obama made us all slaves today. Maybe tomorrow will be a better day....peace to us all... +my sad had work this am smh happy bday tho uce!! I still owe u an sit +I want a dq blizzard. Cheer me up I am sad :( +Sad thing is My parents don't really know who I am.. they know me but dont KNOW me. +my fucking ear hurts, Im sad asf, Im mad asf. ughhh wtf am i gonna do with myself +I don't believe you..fuck ya I am I'm already sad cause I need some alcohol in my system +I am so technology incapable, I cannot use tumbler XD it's so sad lol +The only way im ever gonna get any flowers is if i buy them myself.....and thats sad. http://t.co/ltINc7x3 +i am sad like fucking shit +yeah I know :( sad day +I like tortured souls, sad as I am!! +Well spring break is over in about 1 hour and I am sad:( +are you sad the season is ending? ? I am:( +Just watched the season finally of jersey shore and i am sad its over :( +Is it sad that i am super excited for the new season of Frozen Planet to start Sunday on animal planet.. +(Insert sad asshole tweet here) Yea i wanna cuddle and am going to sleep alone right now... Night bitches! #dueces +dynastycrisis: http://t.co/Z9wSRHqM +When am I going to be able to see him without getting sad.. +we r so sad! i am crying in the first 3 mins ;( +are you sad that the season is ending!? I know I am! +I am very sad to need to set an alarm for the morning. Worst part of getting the boys to school. +Kendra & Lexi .. who am i ? "Oh im crying again:( im so sad.. i really did like him:'( wahhh" .... bitch moan bitch moan.. +But seriously why am I so sad tonight? +When I am sad and mad I pick on my face till I rip flesh off. I need help -_- +As sad as I am that you're not part of my life anymore I'm at least glad someone else is still getting the chance to have you in theirs. ... +me ?? i am so sad read it T.T sorry :'( RT @lovely_leejoon: @dewii_1205 You mean Rani doesn't like you?I see..(T . T) +I'm feeling so sad now... I will no longer be looking forward to #TheWalkingDead every Sunday. What am I going to do? +So sad I have 8 AM class tomorrow #wahwah +And if I am ever sad I feel like everyone thinks its because of you. +I don't remember what this episodes holds!! But you are all so sad, I am worried!! #don'trememberyesterday +I am.. its sad. Like I can't even kill a spidar.. my little cousins do it.. haah +realize you fucking twat. I'm not dragging in a sad way. I am actually quite happy! And I wouldn't say I don't care for you. +Doing the only thing I know to do when I am sad and depressed, eating cheese +O Gawsh please stop, I am over here dying laughin...Sad but, True...! +I dont have gf because girls dont fine me attractive & that is why i am sad,negative girls hate the way i look i hate myself +I wanted to go so bad : c I love them so much it's sad am going to miss out :c sigh* +You're my hero. you inspire me to do good in life. When i am sad I turn to you. I can relate to all of your songs. I love you. +"I'm all out of faith, this is how I feel, sad and I am ashamed, lying without tickets on the floor." I will sing this if I dont get tickets +Ha!!! This is sad, but so true :) honey I am home!!! +I am the #1 fan of #TheWalkingDead I'm just sad to see Shane be killed but Rick had to do it right. +SOUPY MAY NOT BE SAD ANYMORE BUT I SURE AM #fucking #FUCK +it's sad how excited I am for breaking dawn part 2 to come out this november #teamedward #inlove +I am so sad it's almost over. Glad to see Lynette is gonna fight tho!! #getitgirl +When I am sad and think of how amazing my Lord is, it makes me cry tears of happiness. +Ha oh shut up, & I am downnnnn! & are you sad cause soccer season is almost over cause I am :c +I hate to see people sad , so I try my best to make em smile , and it seems to work...thats just how I am....So smile :) +Actually.... Maybe I still AM that sad.... +I am so proud! just sad it had to end how it did. and sad for what I'm gonna have to deal with tomorrow +what is going on? everyone is sad now? I am confused-diana +I am getting soo tired of all the racism and ignorance going on in the world today! The sad thing is #ItNeverEnds +Why am I listening to sad songs? I'm not sad? +After playing DDR and jumping rope today I've come to the sad realization that I am seriously out of shape...yikes. +Getting sad because I am forever alone. xD +I really am just so sad. +Haizzzz.....1 more hours...i am leaving JB.....and my cousin are all leaving ad....SAD!!!!!! +:( its not like i am laying here in bed, not even tired, wishing i had someone to talk too, havent talk to you in days....#SAD +Woot! Thanks chica! I feel like I should be graduating this May, I am doing all of the graduation type things and yet am not. SAD diff --git a/sad_test.txt b/sad_test.txt new file mode 100644 index 0000000..653af2a --- /dev/null +++ b/sad_test.txt @@ -0,0 +1,10 @@ +I am in Nashville at wild horse saloon and see you are playing here can't tell you how bummed I am that I'm missing it! #sad +I am so sad right now +Thank you Lillian, it is very sad for me, but I am excited to see the #yegfood community evolve & support eachother! +I'm am honestly too caring of others & selfless. The sad thing is. I can't do anything bout it. +Did I mention that I am sad I have to go back to work tomorrow....but happy I can just do my job for now on..... +I am sad. Purdue deserved it. #marchmadness +I feel so sick to my stomach. I am so sad. +that's just sad I really am mad now!!!!!! +oh god last pick. do I take Trumbo because I will be sad if I don't or do I take someone else I don't know I am too neurotic for this shit +I am hungry. Wanna cook sup ayam but multi cooker rosak. I have to buy the third multi cooker after this :( Me iz sad.