.gitignore @@ -7,6 +7,7 @@ # YEEZY'S RSS DOES NOT CONTAIN FULLTEXT # WHICH WOULD HAVE MADE THIS MAD EASIER + require 'rubygems' require 'yaml' require 'mechanize' # FOR PARSING KANYEBLOG @@ -14,15 +15,11 @@ require 'open-uri' # FOR BIT.LY MAGIC require 'twitter' # FOR SHOUTING LOUDLY require 'sequel' # FOR STORING FOUND RANTS require 'logger' # FOR CHOPPING WOOD - -# RAPPERS MEET HACKERS? C'MON HELP US OUT -# require 'rss/1.0' -# require 'rss/2.0' -# require 'open-uri' +require 'cgi' # FOR CGI.escape -_- # MIN RANTITTUDE -- AVOID FALSE POSITIVES -MINIMUM_RANT_LENGTH = 200 +MINIMUM_RANT_LENGTH = 500 # WAS THIS TEXT WRITTEN BY KANYE? Y/N @@ -39,27 +36,35 @@ end # SHORTEN A URL USING BIT.LY def bitlyfy(url) + url = CGI.escape(url) return open("http://bit.ly/api?url=#{url}", "UserAgent" => "KANYE RANT DETECTOR <http://fffff.at>").read end -# POST SOMETHING TO TWITTER -def announce(text) +# POST SOMETHING +def announce(msg, url) + + text = "\"#{msg}\": #{url}" puts "ANNOUNCE: #{text}" @config ||= YAML.load(File.open('config.yml')) raise "No config file!" if @config.blank? + + # post to twitter... httpauth = Twitter::HTTPAuth.new(@config['username'], @config['password']) twitter = Twitter::Base.new(httpauth) twitter.update(text) + end -def save(text, url, shorturl, time = DateTime.now) + +# SAVE TO SQLITE +def save(body, excerpt, url, shorturl, time = DateTime.now) rants = DB[:rants] rows = rants.where(:url => url) if rows.blank? || rows.first.nil? puts "NO RECORD! INSERTING NEW..." - return rants.insert(:text => text, :url => url, :shorturl => shorturl, :created_at => time) + return rants.insert(:body => body, :excerpt => excerpt, :url => url, :shorturl => shorturl, :created_at => time) else puts "RECORD ALREADY EXISTS => #{rows.first.inspect}" return false # could not save, already exists @@ -70,7 +75,7 @@ end # ------ WORK IT GIRL -------- -puts "INITIALIZING DATABASE..." +STDERR.puts "INITIALIZING DATABASE..." # INITIALIZE OUR RANTERBASE @@ -80,7 +85,8 @@ unless DB.table_exists?(:rants) primary_key :id String :url String :shorturl - String :text + String :body + String :excerpt DateTime :created_at end end @@ -89,13 +95,17 @@ end # url = "http://www.kanyeuniversecity.com/blog/" # page w/ yesterday's twitter rant # url = "http://www.kanyeuniversecity.com/blog/?em3106=0_-1__-1_~0_-1_5_2009_0_10&em3298=&em3282=&em3281=&em3161=" + # last page... -url = "http://www.kanyeuniversecity.com/blog/?em3106=0_-1__-1_~0_-1_5_2009_0_4820&em3298=&em3282=&em3281=&em3161=" +base = "http://www.kanyeuniversecity.com/blog/" +url = "#{base}?em3106=0_-1__-1_~0_-1_5_2009_0_4820&em3298=&em3282=&em3281=&em3161=" +# REBOOT -> Last page +# url = "http://www.kanyeuniversecity.com/blog/?em3106=0_-1__-1_~0_-1_5_2009_0_4820&em3298=&em3282=&em3281=&em3161=" -puts "CONTACTING INTERNETS... #{url}" +STDERR.puts "CONTACTING INTERNETS... #{url}" agent = WWW::Mechanize.new -agent.user_agent = "KANYE RANT DETECTOR <http://fffff.at>" -# agent.user_agent_alias = "Mac Safari" +# agent.user_agent = "KANYE RANT DETECTOR <http://fffff.at>" +agent.user_agent_alias = "Mac Safari" page = agent.get(url) # FOR PROPER ARCHIVAL @@ -104,12 +114,17 @@ reverse_pagination = true # DETECT KANYES GOGOGOGO first = 0 # GETS OVERRIDDEN loop { - (page/'.rapper').each { |post| - - excerpt = post.content[0..100].gsub("\n",'').gsub("\t",'').chomp - permalink = url+(post/'a').first['href'] + # IN REVERSE MODE... + (page/'.rapper').to_a.reverse.each { |post| + + content = post.content.strip! + excerpt = post.content[0..120].gsub("\n",'').gsub("\t",'').strip! + + links = (post/'a') + permalink = "#{base}#{links[0]['href']}" text = (post/'h5').first.content.strip_html - puts "PROCESSING: #{excerpt} ..." + # puts "PROCESSING: #{excerpt}" + # puts "#{permalink}" # FOUND A SHORTCUT: ONLY RANTS ARE IN SPECIFIED ELEMENT content = (post/'h5 div').first.content rescue '' @@ -117,14 +132,13 @@ loop { # TELL THE MAFACKIN WORLD # BONUS: DO IT WITH AUTOTUNE if !content.empty? && content.length > MINIMUM_RANT_LENGTH - shorturl = bitlyfy(permalink) - msg = "KANYERANT! \"#{excerpt}\": #{shorturl}" + shorturl = bitlyfy(permalink) # ONLY ANNOUNCE ON SUCCESFUL SAVE TO DB - announce(msg) if save(msg, permalink, shorturl) - puts "..." + announce(excerpt, shorturl) if save(content, excerpt, permalink, shorturl) + puts "" - sleep 5 + sleep 3 end @@ -133,14 +147,23 @@ loop { # RECURSE PAGES... THEY POST A LOT current = (page/'#emodpages strong')[1].content.to_i first = current if first == 0 - puts "current = #{current.inspect} -- first = #{first.inspect}" + # puts "current = #{current.inspect} -- first = #{first.inspect}" prev = (page/'#emodpages a').select { |e| e.content.strip_html.to_i == (reverse_pagination ? current - 1 : current + 1) } - puts "prev = #{prev}" - break if prev.blank? + if prev.blank? or prev.first.nil? + STDERR.puts "No prev link!" + break + end + + sleep 1 + link = prev.first + pagenum = link.content.strip_html.to_i + if pagenum.blank? || pagenum < 1 + STDERR.puts "Pagenum #{pagenum} is the end of the road! We're done here" + break + end - sleep 3 - page = agent.click(prev.first) - puts "---- loaded page #{prev} -----" + page = agent.click(link) + puts "---- loaded page #{pagenum} ----- #{link['href']}" } RANT-DETECTOR.rb kanyerants.db 18a6312f8a87f0963de0786f58e51aff6ccbb07f Jamie Wilkinson jamie@tramchase.com http://github.com/jamiew/KANYE-RANT-DETECTOR/commit/f5f167a4fdbbeca85d794f6f3ed1783a26d4e450 f5f167a4fdbbeca85d794f6f3ed1783a26d4e450 2009-05-14T07:43:53-07:00 2009-05-14T07:43:53-07:00 REVERSE CHRONO KANYE PARSING, FOR ARCHIVAL PURPOSES. TURNED UP RANT THRESHOLD, HAD SOME FALSE POSITIVES IN THE RUN LAST NIGHT. ALL LOOKS WELL IN THE KINGDOM... f783d3a5bcc67a89738843f4681afd55b3621645 Jamie Wilkinson jamie@tramchase.com