Skip to content
Browse files

Going to add persistent job store

  • Loading branch information...
1 parent 4cfcbb8 commit 328e1d6a11b3562e6fb5ee1481b8ca63283643e6 Philip (flip) Kromer committed
View
1 distdb
View
6 examples/scrape/twitter/twitter_search_jobs.tsv
@@ -0,0 +1,6 @@
+love 65536 3.36366494448618 6698 2686693027 2688691633
+red+sox 65536 0.0113457581992013 1500 2661001994 2688059232
+britney+spears 65536 0.00866753886170806 184 2685103763 2688130850
+hadoop 65536 0.000661831916251315 614 2501794487 2687967783
+infochimps 65536 2.24964286919452e-05 16 2541533220 2683708276
+hapaxlegomenon 65536 0.0 1 2646535741 2646535741
View
725 examples/scrape/twitter/twitter_trending_seed.tsv
@@ -0,0 +1,725 @@
+http://search.twitter.com/trends/weekly.json?date=2008-11-19
+http://search.twitter.com/trends/weekly.json?date=2008-11-20
+http://search.twitter.com/trends/weekly.json?date=2008-11-21
+http://search.twitter.com/trends/weekly.json?date=2008-11-22
+http://search.twitter.com/trends/weekly.json?date=2008-11-23
+http://search.twitter.com/trends/weekly.json?date=2008-11-24
+http://search.twitter.com/trends/weekly.json?date=2008-11-25
+http://search.twitter.com/trends/weekly.json?date=2008-11-26
+http://search.twitter.com/trends/weekly.json?date=2008-11-27
+http://search.twitter.com/trends/weekly.json?date=2008-11-28
+http://search.twitter.com/trends/weekly.json?date=2008-11-29
+http://search.twitter.com/trends/weekly.json?date=2008-11-30
+http://search.twitter.com/trends/weekly.json?date=2008-12-01
+http://search.twitter.com/trends/weekly.json?date=2008-12-02
+http://search.twitter.com/trends/weekly.json?date=2008-12-03
+http://search.twitter.com/trends/weekly.json?date=2008-12-04
+http://search.twitter.com/trends/weekly.json?date=2008-12-05
+http://search.twitter.com/trends/weekly.json?date=2008-12-06
+http://search.twitter.com/trends/weekly.json?date=2008-12-07
+http://search.twitter.com/trends/weekly.json?date=2008-12-08
+http://search.twitter.com/trends/weekly.json?date=2008-12-09
+http://search.twitter.com/trends/weekly.json?date=2008-12-10
+http://search.twitter.com/trends/weekly.json?date=2008-12-11
+http://search.twitter.com/trends/weekly.json?date=2008-12-12
+http://search.twitter.com/trends/weekly.json?date=2008-12-13
+http://search.twitter.com/trends/weekly.json?date=2008-12-14
+http://search.twitter.com/trends/weekly.json?date=2008-12-15
+http://search.twitter.com/trends/weekly.json?date=2008-12-16
+http://search.twitter.com/trends/weekly.json?date=2008-12-17
+http://search.twitter.com/trends/weekly.json?date=2008-12-18
+http://search.twitter.com/trends/weekly.json?date=2008-12-19
+http://search.twitter.com/trends/weekly.json?date=2008-12-20
+http://search.twitter.com/trends/weekly.json?date=2008-12-21
+http://search.twitter.com/trends/weekly.json?date=2008-12-22
+http://search.twitter.com/trends/weekly.json?date=2008-12-23
+http://search.twitter.com/trends/weekly.json?date=2008-12-24
+http://search.twitter.com/trends/weekly.json?date=2008-12-25
+http://search.twitter.com/trends/weekly.json?date=2008-12-26
+http://search.twitter.com/trends/weekly.json?date=2008-12-27
+http://search.twitter.com/trends/weekly.json?date=2008-12-28
+http://search.twitter.com/trends/weekly.json?date=2008-12-29
+http://search.twitter.com/trends/weekly.json?date=2008-12-30
+http://search.twitter.com/trends/weekly.json?date=2008-12-31
+http://search.twitter.com/trends/weekly.json?date=2009-01-01
+http://search.twitter.com/trends/weekly.json?date=2009-01-02
+http://search.twitter.com/trends/weekly.json?date=2009-01-03
+http://search.twitter.com/trends/weekly.json?date=2009-01-04
+http://search.twitter.com/trends/weekly.json?date=2009-01-05
+http://search.twitter.com/trends/weekly.json?date=2009-01-06
+http://search.twitter.com/trends/weekly.json?date=2009-01-07
+http://search.twitter.com/trends/weekly.json?date=2009-01-08
+http://search.twitter.com/trends/weekly.json?date=2009-01-09
+http://search.twitter.com/trends/weekly.json?date=2009-01-10
+http://search.twitter.com/trends/weekly.json?date=2009-01-11
+http://search.twitter.com/trends/weekly.json?date=2009-01-12
+http://search.twitter.com/trends/weekly.json?date=2009-01-13
+http://search.twitter.com/trends/weekly.json?date=2009-01-14
+http://search.twitter.com/trends/weekly.json?date=2009-01-15
+http://search.twitter.com/trends/weekly.json?date=2009-01-16
+http://search.twitter.com/trends/weekly.json?date=2009-01-17
+http://search.twitter.com/trends/weekly.json?date=2009-01-18
+http://search.twitter.com/trends/weekly.json?date=2009-01-19
+http://search.twitter.com/trends/weekly.json?date=2009-01-20
+http://search.twitter.com/trends/weekly.json?date=2009-01-21
+http://search.twitter.com/trends/weekly.json?date=2009-01-22
+http://search.twitter.com/trends/weekly.json?date=2009-01-23
+http://search.twitter.com/trends/weekly.json?date=2009-01-24
+http://search.twitter.com/trends/weekly.json?date=2009-01-25
+http://search.twitter.com/trends/weekly.json?date=2009-01-26
+http://search.twitter.com/trends/weekly.json?date=2009-01-27
+http://search.twitter.com/trends/weekly.json?date=2009-01-28
+http://search.twitter.com/trends/weekly.json?date=2009-01-29
+http://search.twitter.com/trends/weekly.json?date=2009-01-30
+http://search.twitter.com/trends/weekly.json?date=2009-01-31
+http://search.twitter.com/trends/weekly.json?date=2009-02-01
+http://search.twitter.com/trends/weekly.json?date=2009-02-02
+http://search.twitter.com/trends/weekly.json?date=2009-02-03
+http://search.twitter.com/trends/weekly.json?date=2009-02-04
+http://search.twitter.com/trends/weekly.json?date=2009-02-05
+http://search.twitter.com/trends/weekly.json?date=2009-02-06
+http://search.twitter.com/trends/weekly.json?date=2009-02-07
+http://search.twitter.com/trends/weekly.json?date=2009-02-08
+http://search.twitter.com/trends/weekly.json?date=2009-02-09
+http://search.twitter.com/trends/weekly.json?date=2009-02-10
+http://search.twitter.com/trends/weekly.json?date=2009-02-11
+http://search.twitter.com/trends/weekly.json?date=2009-02-12
+http://search.twitter.com/trends/weekly.json?date=2009-02-13
+http://search.twitter.com/trends/weekly.json?date=2009-02-14
+http://search.twitter.com/trends/weekly.json?date=2009-02-15
+http://search.twitter.com/trends/weekly.json?date=2009-02-16
+http://search.twitter.com/trends/weekly.json?date=2009-02-17
+http://search.twitter.com/trends/weekly.json?date=2009-02-18
+http://search.twitter.com/trends/weekly.json?date=2009-02-19
+http://search.twitter.com/trends/weekly.json?date=2009-02-20
+http://search.twitter.com/trends/weekly.json?date=2009-02-21
+http://search.twitter.com/trends/weekly.json?date=2009-02-22
+http://search.twitter.com/trends/weekly.json?date=2009-02-23
+http://search.twitter.com/trends/weekly.json?date=2009-02-24
+http://search.twitter.com/trends/weekly.json?date=2009-02-25
+http://search.twitter.com/trends/weekly.json?date=2009-02-26
+http://search.twitter.com/trends/weekly.json?date=2009-02-27
+http://search.twitter.com/trends/weekly.json?date=2009-02-28
+http://search.twitter.com/trends/weekly.json?date=2009-03-01
+http://search.twitter.com/trends/weekly.json?date=2009-03-02
+http://search.twitter.com/trends/weekly.json?date=2009-03-03
+http://search.twitter.com/trends/weekly.json?date=2009-03-04
+http://search.twitter.com/trends/weekly.json?date=2009-03-05
+http://search.twitter.com/trends/weekly.json?date=2009-03-06
+http://search.twitter.com/trends/weekly.json?date=2009-03-07
+http://search.twitter.com/trends/weekly.json?date=2009-03-08
+http://search.twitter.com/trends/weekly.json?date=2009-03-09
+http://search.twitter.com/trends/weekly.json?date=2009-03-10
+http://search.twitter.com/trends/weekly.json?date=2009-03-11
+http://search.twitter.com/trends/weekly.json?date=2009-03-12
+http://search.twitter.com/trends/weekly.json?date=2009-03-13
+http://search.twitter.com/trends/weekly.json?date=2009-03-14
+http://search.twitter.com/trends/weekly.json?date=2009-03-15
+http://search.twitter.com/trends/weekly.json?date=2009-03-16
+http://search.twitter.com/trends/weekly.json?date=2009-03-17
+http://search.twitter.com/trends/weekly.json?date=2009-03-18
+http://search.twitter.com/trends/weekly.json?date=2009-03-19
+http://search.twitter.com/trends/weekly.json?date=2009-03-20
+http://search.twitter.com/trends/weekly.json?date=2009-03-21
+http://search.twitter.com/trends/weekly.json?date=2009-03-22
+http://search.twitter.com/trends/weekly.json?date=2009-03-23
+http://search.twitter.com/trends/weekly.json?date=2009-03-24
+http://search.twitter.com/trends/weekly.json?date=2009-03-25
+http://search.twitter.com/trends/weekly.json?date=2009-03-26
+http://search.twitter.com/trends/weekly.json?date=2009-03-27
+http://search.twitter.com/trends/weekly.json?date=2009-03-28
+http://search.twitter.com/trends/weekly.json?date=2009-03-29
+http://search.twitter.com/trends/weekly.json?date=2009-03-30
+http://search.twitter.com/trends/weekly.json?date=2009-03-31
+http://search.twitter.com/trends/weekly.json?date=2009-04-01
+http://search.twitter.com/trends/weekly.json?date=2009-04-02
+http://search.twitter.com/trends/weekly.json?date=2009-04-03
+http://search.twitter.com/trends/weekly.json?date=2009-04-04
+http://search.twitter.com/trends/weekly.json?date=2009-04-05
+http://search.twitter.com/trends/weekly.json?date=2009-04-06
+http://search.twitter.com/trends/weekly.json?date=2009-04-07
+http://search.twitter.com/trends/weekly.json?date=2009-04-08
+http://search.twitter.com/trends/weekly.json?date=2009-04-09
+http://search.twitter.com/trends/weekly.json?date=2009-04-10
+http://search.twitter.com/trends/weekly.json?date=2009-04-11
+http://search.twitter.com/trends/weekly.json?date=2009-04-12
+http://search.twitter.com/trends/weekly.json?date=2009-04-13
+http://search.twitter.com/trends/weekly.json?date=2009-04-14
+http://search.twitter.com/trends/weekly.json?date=2009-04-15
+http://search.twitter.com/trends/weekly.json?date=2009-04-16
+http://search.twitter.com/trends/weekly.json?date=2009-04-17
+http://search.twitter.com/trends/weekly.json?date=2009-04-18
+http://search.twitter.com/trends/weekly.json?date=2009-04-19
+http://search.twitter.com/trends/weekly.json?date=2009-04-20
+http://search.twitter.com/trends/weekly.json?date=2009-04-21
+http://search.twitter.com/trends/weekly.json?date=2009-04-22
+http://search.twitter.com/trends/weekly.json?date=2009-04-23
+http://search.twitter.com/trends/weekly.json?date=2009-04-24
+http://search.twitter.com/trends/weekly.json?date=2009-04-25
+http://search.twitter.com/trends/weekly.json?date=2009-04-26
+http://search.twitter.com/trends/weekly.json?date=2009-04-27
+http://search.twitter.com/trends/weekly.json?date=2009-04-28
+http://search.twitter.com/trends/weekly.json?date=2009-04-29
+http://search.twitter.com/trends/weekly.json?date=2009-04-30
+http://search.twitter.com/trends/weekly.json?date=2009-05-01
+http://search.twitter.com/trends/weekly.json?date=2009-05-02
+http://search.twitter.com/trends/weekly.json?date=2009-05-03
+http://search.twitter.com/trends/weekly.json?date=2009-05-04
+http://search.twitter.com/trends/weekly.json?date=2009-05-05
+http://search.twitter.com/trends/weekly.json?date=2009-05-06
+http://search.twitter.com/trends/weekly.json?date=2009-05-07
+http://search.twitter.com/trends/weekly.json?date=2009-05-08
+http://search.twitter.com/trends/weekly.json?date=2009-05-09
+http://search.twitter.com/trends/weekly.json?date=2009-05-10
+http://search.twitter.com/trends/weekly.json?date=2009-05-11
+http://search.twitter.com/trends/weekly.json?date=2009-05-12
+http://search.twitter.com/trends/weekly.json?date=2009-05-13
+http://search.twitter.com/trends/weekly.json?date=2009-05-14
+http://search.twitter.com/trends/weekly.json?date=2009-05-15
+http://search.twitter.com/trends/weekly.json?date=2009-05-16
+http://search.twitter.com/trends/weekly.json?date=2009-05-17
+http://search.twitter.com/trends/weekly.json?date=2009-05-18
+http://search.twitter.com/trends/weekly.json?date=2009-05-19
+http://search.twitter.com/trends/weekly.json?date=2009-05-20
+http://search.twitter.com/trends/weekly.json?date=2009-05-21
+http://search.twitter.com/trends/weekly.json?date=2009-05-22
+http://search.twitter.com/trends/weekly.json?date=2009-05-23
+http://search.twitter.com/trends/weekly.json?date=2009-05-24
+http://search.twitter.com/trends/weekly.json?date=2009-05-25
+http://search.twitter.com/trends/weekly.json?date=2009-05-26
+http://search.twitter.com/trends/weekly.json?date=2009-05-27
+http://search.twitter.com/trends/weekly.json?date=2009-05-28
+http://search.twitter.com/trends/weekly.json?date=2009-05-29
+http://search.twitter.com/trends/weekly.json?date=2009-05-30
+http://search.twitter.com/trends/weekly.json?date=2009-05-31
+http://search.twitter.com/trends/weekly.json?date=2009-06-01
+http://search.twitter.com/trends/weekly.json?date=2009-06-02
+http://search.twitter.com/trends/weekly.json?date=2009-06-03
+http://search.twitter.com/trends/weekly.json?date=2009-06-04
+http://search.twitter.com/trends/weekly.json?date=2009-06-05
+http://search.twitter.com/trends/weekly.json?date=2009-06-06
+http://search.twitter.com/trends/weekly.json?date=2009-06-07
+http://search.twitter.com/trends/weekly.json?date=2009-06-08
+http://search.twitter.com/trends/weekly.json?date=2009-06-09
+http://search.twitter.com/trends/weekly.json?date=2009-06-10
+http://search.twitter.com/trends/weekly.json?date=2009-06-11
+http://search.twitter.com/trends/weekly.json?date=2009-06-12
+http://search.twitter.com/trends/weekly.json?date=2009-06-13
+http://search.twitter.com/trends/weekly.json?date=2009-06-14
+http://search.twitter.com/trends/weekly.json?date=2009-06-15
+http://search.twitter.com/trends/weekly.json?date=2009-06-16
+http://search.twitter.com/trends/weekly.json?date=2009-06-17
+http://search.twitter.com/trends/weekly.json?date=2009-06-18
+http://search.twitter.com/trends/weekly.json?date=2009-06-19
+http://search.twitter.com/trends/weekly.json?date=2009-06-20
+http://search.twitter.com/trends/weekly.json?date=2009-06-21
+http://search.twitter.com/trends/weekly.json?date=2009-06-22
+http://search.twitter.com/trends/weekly.json?date=2009-06-23
+http://search.twitter.com/trends/weekly.json?date=2009-06-24
+http://search.twitter.com/trends/weekly.json?date=2009-06-25
+http://search.twitter.com/trends/weekly.json?date=2009-06-26
+http://search.twitter.com/trends/weekly.json?date=2009-06-27
+http://search.twitter.com/trends/weekly.json?date=2009-06-28
+http://search.twitter.com/trends/weekly.json?date=2009-06-29
+http://search.twitter.com/trends/weekly.json?date=2009-06-30
+http://search.twitter.com/trends/weekly.json?date=2009-07-01
+http://search.twitter.com/trends/weekly.json?date=2009-07-02
+http://search.twitter.com/trends/weekly.json?date=2009-07-03
+http://search.twitter.com/trends/weekly.json?date=2009-07-04
+http://search.twitter.com/trends/weekly.json?date=2009-07-05
+http://search.twitter.com/trends/weekly.json?date=2009-07-06
+http://search.twitter.com/trends/weekly.json?date=2009-07-07
+http://search.twitter.com/trends/weekly.json?date=2009-07-08
+http://search.twitter.com/trends/weekly.json?date=2009-07-09
+http://search.twitter.com/trends/weekly.json?date=2009-07-10
+http://search.twitter.com/trends/weekly.json?date=2009-07-11
+http://search.twitter.com/trends/weekly.json?date=2009-07-12
+http://search.twitter.com/trends/weekly.json?date=2009-07-13
+http://search.twitter.com/trends/weekly.json?date=2009-07-14
+http://search.twitter.com/trends/weekly.json?date=2009-07-15
+http://search.twitter.com/trends/weekly.json?date=2009-07-16
+http://search.twitter.com/trends/weekly.json?date=2009-07-17
+=> Wed, 19 Nov 2008 00:00:00 0000..Fri, 17 Jul 2009 10:01:48 -0500
+>> (DateTime.parse('2008-11-19') .. DateTime.now).each do |date| puts "http://search.twitter.com/trends/weekly.json?date=#{date.strftime("%Y-%m-%d")}" end ; (DateTime.parse('2008-11-19') .. DateTime.now).each do |date| puts "http://search.twitter.com/trends/daily.json?date=#{date.strftime("%Y-%m-%d")}" end
+http://search.twitter.com/trends/weekly.json?date=2008-11-19
+http://search.twitter.com/trends/weekly.json?date=2008-11-20
+http://search.twitter.com/trends/weekly.json?date=2008-11-21
+http://search.twitter.com/trends/weekly.json?date=2008-11-22
+http://search.twitter.com/trends/weekly.json?date=2008-11-23
+http://search.twitter.com/trends/weekly.json?date=2008-11-24
+http://search.twitter.com/trends/weekly.json?date=2008-11-25
+http://search.twitter.com/trends/weekly.json?date=2008-11-26
+http://search.twitter.com/trends/weekly.json?date=2008-11-27
+http://search.twitter.com/trends/weekly.json?date=2008-11-28
+http://search.twitter.com/trends/weekly.json?date=2008-11-29
+http://search.twitter.com/trends/weekly.json?date=2008-11-30
+http://search.twitter.com/trends/weekly.json?date=2008-12-01
+http://search.twitter.com/trends/weekly.json?date=2008-12-02
+http://search.twitter.com/trends/weekly.json?date=2008-12-03
+http://search.twitter.com/trends/weekly.json?date=2008-12-04
+http://search.twitter.com/trends/weekly.json?date=2008-12-05
+http://search.twitter.com/trends/weekly.json?date=2008-12-06
+http://search.twitter.com/trends/weekly.json?date=2008-12-07
+http://search.twitter.com/trends/weekly.json?date=2008-12-08
+http://search.twitter.com/trends/weekly.json?date=2008-12-09
+http://search.twitter.com/trends/weekly.json?date=2008-12-10
+http://search.twitter.com/trends/weekly.json?date=2008-12-11
+http://search.twitter.com/trends/weekly.json?date=2008-12-12
+http://search.twitter.com/trends/weekly.json?date=2008-12-13
+http://search.twitter.com/trends/weekly.json?date=2008-12-14
+http://search.twitter.com/trends/weekly.json?date=2008-12-15
+http://search.twitter.com/trends/weekly.json?date=2008-12-16
+http://search.twitter.com/trends/weekly.json?date=2008-12-17
+http://search.twitter.com/trends/weekly.json?date=2008-12-18
+http://search.twitter.com/trends/weekly.json?date=2008-12-19
+http://search.twitter.com/trends/weekly.json?date=2008-12-20
+http://search.twitter.com/trends/weekly.json?date=2008-12-21
+http://search.twitter.com/trends/weekly.json?date=2008-12-22
+http://search.twitter.com/trends/weekly.json?date=2008-12-23
+http://search.twitter.com/trends/weekly.json?date=2008-12-24
+http://search.twitter.com/trends/weekly.json?date=2008-12-25
+http://search.twitter.com/trends/weekly.json?date=2008-12-26
+http://search.twitter.com/trends/weekly.json?date=2008-12-27
+http://search.twitter.com/trends/weekly.json?date=2008-12-28
+http://search.twitter.com/trends/weekly.json?date=2008-12-29
+http://search.twitter.com/trends/weekly.json?date=2008-12-30
+http://search.twitter.com/trends/weekly.json?date=2008-12-31
+http://search.twitter.com/trends/weekly.json?date=2009-01-01
+http://search.twitter.com/trends/weekly.json?date=2009-01-02
+http://search.twitter.com/trends/weekly.json?date=2009-01-03
+http://search.twitter.com/trends/weekly.json?date=2009-01-04
+http://search.twitter.com/trends/weekly.json?date=2009-01-05
+http://search.twitter.com/trends/weekly.json?date=2009-01-06
+http://search.twitter.com/trends/weekly.json?date=2009-01-07
+http://search.twitter.com/trends/weekly.json?date=2009-01-08
+http://search.twitter.com/trends/weekly.json?date=2009-01-09
+http://search.twitter.com/trends/weekly.json?date=2009-01-10
+http://search.twitter.com/trends/weekly.json?date=2009-01-11
+http://search.twitter.com/trends/weekly.json?date=2009-01-12
+http://search.twitter.com/trends/weekly.json?date=2009-01-13
+http://search.twitter.com/trends/weekly.json?date=2009-01-14
+http://search.twitter.com/trends/weekly.json?date=2009-01-15
+http://search.twitter.com/trends/weekly.json?date=2009-01-16
+http://search.twitter.com/trends/weekly.json?date=2009-01-17
+http://search.twitter.com/trends/weekly.json?date=2009-01-18
+http://search.twitter.com/trends/weekly.json?date=2009-01-19
+http://search.twitter.com/trends/weekly.json?date=2009-01-20
+http://search.twitter.com/trends/weekly.json?date=2009-01-21
+http://search.twitter.com/trends/weekly.json?date=2009-01-22
+http://search.twitter.com/trends/weekly.json?date=2009-01-23
+http://search.twitter.com/trends/weekly.json?date=2009-01-24
+http://search.twitter.com/trends/weekly.json?date=2009-01-25
+http://search.twitter.com/trends/weekly.json?date=2009-01-26
+http://search.twitter.com/trends/weekly.json?date=2009-01-27
+http://search.twitter.com/trends/weekly.json?date=2009-01-28
+http://search.twitter.com/trends/weekly.json?date=2009-01-29
+http://search.twitter.com/trends/weekly.json?date=2009-01-30
+http://search.twitter.com/trends/weekly.json?date=2009-01-31
+http://search.twitter.com/trends/weekly.json?date=2009-02-01
+http://search.twitter.com/trends/weekly.json?date=2009-02-02
+http://search.twitter.com/trends/weekly.json?date=2009-02-03
+http://search.twitter.com/trends/weekly.json?date=2009-02-04
+http://search.twitter.com/trends/weekly.json?date=2009-02-05
+http://search.twitter.com/trends/weekly.json?date=2009-02-06
+http://search.twitter.com/trends/weekly.json?date=2009-02-07
+http://search.twitter.com/trends/weekly.json?date=2009-02-08
+http://search.twitter.com/trends/weekly.json?date=2009-02-09
+http://search.twitter.com/trends/weekly.json?date=2009-02-10
+http://search.twitter.com/trends/weekly.json?date=2009-02-11
+http://search.twitter.com/trends/weekly.json?date=2009-02-12
+http://search.twitter.com/trends/weekly.json?date=2009-02-13
+http://search.twitter.com/trends/weekly.json?date=2009-02-14
+http://search.twitter.com/trends/weekly.json?date=2009-02-15
+http://search.twitter.com/trends/weekly.json?date=2009-02-16
+http://search.twitter.com/trends/weekly.json?date=2009-02-17
+http://search.twitter.com/trends/weekly.json?date=2009-02-18
+http://search.twitter.com/trends/weekly.json?date=2009-02-19
+http://search.twitter.com/trends/weekly.json?date=2009-02-20
+http://search.twitter.com/trends/weekly.json?date=2009-02-21
+http://search.twitter.com/trends/weekly.json?date=2009-02-22
+http://search.twitter.com/trends/weekly.json?date=2009-02-23
+http://search.twitter.com/trends/weekly.json?date=2009-02-24
+http://search.twitter.com/trends/weekly.json?date=2009-02-25
+http://search.twitter.com/trends/weekly.json?date=2009-02-26
+http://search.twitter.com/trends/weekly.json?date=2009-02-27
+http://search.twitter.com/trends/weekly.json?date=2009-02-28
+http://search.twitter.com/trends/weekly.json?date=2009-03-01
+http://search.twitter.com/trends/weekly.json?date=2009-03-02
+http://search.twitter.com/trends/weekly.json?date=2009-03-03
+http://search.twitter.com/trends/weekly.json?date=2009-03-04
+http://search.twitter.com/trends/weekly.json?date=2009-03-05
+http://search.twitter.com/trends/weekly.json?date=2009-03-06
+http://search.twitter.com/trends/weekly.json?date=2009-03-07
+http://search.twitter.com/trends/weekly.json?date=2009-03-08
+http://search.twitter.com/trends/weekly.json?date=2009-03-09
+http://search.twitter.com/trends/weekly.json?date=2009-03-10
+http://search.twitter.com/trends/weekly.json?date=2009-03-11
+http://search.twitter.com/trends/weekly.json?date=2009-03-12
+http://search.twitter.com/trends/weekly.json?date=2009-03-13
+http://search.twitter.com/trends/weekly.json?date=2009-03-14
+http://search.twitter.com/trends/weekly.json?date=2009-03-15
+http://search.twitter.com/trends/weekly.json?date=2009-03-16
+http://search.twitter.com/trends/weekly.json?date=2009-03-17
+http://search.twitter.com/trends/weekly.json?date=2009-03-18
+http://search.twitter.com/trends/weekly.json?date=2009-03-19
+http://search.twitter.com/trends/weekly.json?date=2009-03-20
+http://search.twitter.com/trends/weekly.json?date=2009-03-21
+http://search.twitter.com/trends/weekly.json?date=2009-03-22
+http://search.twitter.com/trends/weekly.json?date=2009-03-23
+http://search.twitter.com/trends/weekly.json?date=2009-03-24
+http://search.twitter.com/trends/weekly.json?date=2009-03-25
+http://search.twitter.com/trends/weekly.json?date=2009-03-26
+http://search.twitter.com/trends/weekly.json?date=2009-03-27
+http://search.twitter.com/trends/weekly.json?date=2009-03-28
+http://search.twitter.com/trends/weekly.json?date=2009-03-29
+http://search.twitter.com/trends/weekly.json?date=2009-03-30
+http://search.twitter.com/trends/weekly.json?date=2009-03-31
+http://search.twitter.com/trends/weekly.json?date=2009-04-01
+http://search.twitter.com/trends/weekly.json?date=2009-04-02
+http://search.twitter.com/trends/weekly.json?date=2009-04-03
+http://search.twitter.com/trends/weekly.json?date=2009-04-04
+http://search.twitter.com/trends/weekly.json?date=2009-04-05
+http://search.twitter.com/trends/weekly.json?date=2009-04-06
+http://search.twitter.com/trends/weekly.json?date=2009-04-07
+http://search.twitter.com/trends/weekly.json?date=2009-04-08
+http://search.twitter.com/trends/weekly.json?date=2009-04-09
+http://search.twitter.com/trends/weekly.json?date=2009-04-10
+http://search.twitter.com/trends/weekly.json?date=2009-04-11
+http://search.twitter.com/trends/weekly.json?date=2009-04-12
+http://search.twitter.com/trends/weekly.json?date=2009-04-13
+http://search.twitter.com/trends/weekly.json?date=2009-04-14
+http://search.twitter.com/trends/weekly.json?date=2009-04-15
+http://search.twitter.com/trends/weekly.json?date=2009-04-16
+http://search.twitter.com/trends/weekly.json?date=2009-04-17
+http://search.twitter.com/trends/weekly.json?date=2009-04-18
+http://search.twitter.com/trends/weekly.json?date=2009-04-19
+http://search.twitter.com/trends/weekly.json?date=2009-04-20
+http://search.twitter.com/trends/weekly.json?date=2009-04-21
+http://search.twitter.com/trends/weekly.json?date=2009-04-22
+http://search.twitter.com/trends/weekly.json?date=2009-04-23
+http://search.twitter.com/trends/weekly.json?date=2009-04-24
+http://search.twitter.com/trends/weekly.json?date=2009-04-25
+http://search.twitter.com/trends/weekly.json?date=2009-04-26
+http://search.twitter.com/trends/weekly.json?date=2009-04-27
+http://search.twitter.com/trends/weekly.json?date=2009-04-28
+http://search.twitter.com/trends/weekly.json?date=2009-04-29
+http://search.twitter.com/trends/weekly.json?date=2009-04-30
+http://search.twitter.com/trends/weekly.json?date=2009-05-01
+http://search.twitter.com/trends/weekly.json?date=2009-05-02
+http://search.twitter.com/trends/weekly.json?date=2009-05-03
+http://search.twitter.com/trends/weekly.json?date=2009-05-04
+http://search.twitter.com/trends/weekly.json?date=2009-05-05
+http://search.twitter.com/trends/weekly.json?date=2009-05-06
+http://search.twitter.com/trends/weekly.json?date=2009-05-07
+http://search.twitter.com/trends/weekly.json?date=2009-05-08
+http://search.twitter.com/trends/weekly.json?date=2009-05-09
+http://search.twitter.com/trends/weekly.json?date=2009-05-10
+http://search.twitter.com/trends/weekly.json?date=2009-05-11
+http://search.twitter.com/trends/weekly.json?date=2009-05-12
+http://search.twitter.com/trends/weekly.json?date=2009-05-13
+http://search.twitter.com/trends/weekly.json?date=2009-05-14
+http://search.twitter.com/trends/weekly.json?date=2009-05-15
+http://search.twitter.com/trends/weekly.json?date=2009-05-16
+http://search.twitter.com/trends/weekly.json?date=2009-05-17
+http://search.twitter.com/trends/weekly.json?date=2009-05-18
+http://search.twitter.com/trends/weekly.json?date=2009-05-19
+http://search.twitter.com/trends/weekly.json?date=2009-05-20
+http://search.twitter.com/trends/weekly.json?date=2009-05-21
+http://search.twitter.com/trends/weekly.json?date=2009-05-22
+http://search.twitter.com/trends/weekly.json?date=2009-05-23
+http://search.twitter.com/trends/weekly.json?date=2009-05-24
+http://search.twitter.com/trends/weekly.json?date=2009-05-25
+http://search.twitter.com/trends/weekly.json?date=2009-05-26
+http://search.twitter.com/trends/weekly.json?date=2009-05-27
+http://search.twitter.com/trends/weekly.json?date=2009-05-28
+http://search.twitter.com/trends/weekly.json?date=2009-05-29
+http://search.twitter.com/trends/weekly.json?date=2009-05-30
+http://search.twitter.com/trends/weekly.json?date=2009-05-31
+http://search.twitter.com/trends/weekly.json?date=2009-06-01
+http://search.twitter.com/trends/weekly.json?date=2009-06-02
+http://search.twitter.com/trends/weekly.json?date=2009-06-03
+http://search.twitter.com/trends/weekly.json?date=2009-06-04
+http://search.twitter.com/trends/weekly.json?date=2009-06-05
+http://search.twitter.com/trends/weekly.json?date=2009-06-06
+http://search.twitter.com/trends/weekly.json?date=2009-06-07
+http://search.twitter.com/trends/weekly.json?date=2009-06-08
+http://search.twitter.com/trends/weekly.json?date=2009-06-09
+http://search.twitter.com/trends/weekly.json?date=2009-06-10
+http://search.twitter.com/trends/weekly.json?date=2009-06-11
+http://search.twitter.com/trends/weekly.json?date=2009-06-12
+http://search.twitter.com/trends/weekly.json?date=2009-06-13
+http://search.twitter.com/trends/weekly.json?date=2009-06-14
+http://search.twitter.com/trends/weekly.json?date=2009-06-15
+http://search.twitter.com/trends/weekly.json?date=2009-06-16
+http://search.twitter.com/trends/weekly.json?date=2009-06-17
+http://search.twitter.com/trends/weekly.json?date=2009-06-18
+http://search.twitter.com/trends/weekly.json?date=2009-06-19
+http://search.twitter.com/trends/weekly.json?date=2009-06-20
+http://search.twitter.com/trends/weekly.json?date=2009-06-21
+http://search.twitter.com/trends/weekly.json?date=2009-06-22
+http://search.twitter.com/trends/weekly.json?date=2009-06-23
+http://search.twitter.com/trends/weekly.json?date=2009-06-24
+http://search.twitter.com/trends/weekly.json?date=2009-06-25
+http://search.twitter.com/trends/weekly.json?date=2009-06-26
+http://search.twitter.com/trends/weekly.json?date=2009-06-27
+http://search.twitter.com/trends/weekly.json?date=2009-06-28
+http://search.twitter.com/trends/weekly.json?date=2009-06-29
+http://search.twitter.com/trends/weekly.json?date=2009-06-30
+http://search.twitter.com/trends/weekly.json?date=2009-07-01
+http://search.twitter.com/trends/weekly.json?date=2009-07-02
+http://search.twitter.com/trends/weekly.json?date=2009-07-03
+http://search.twitter.com/trends/weekly.json?date=2009-07-04
+http://search.twitter.com/trends/weekly.json?date=2009-07-05
+http://search.twitter.com/trends/weekly.json?date=2009-07-06
+http://search.twitter.com/trends/weekly.json?date=2009-07-07
+http://search.twitter.com/trends/weekly.json?date=2009-07-08
+http://search.twitter.com/trends/weekly.json?date=2009-07-09
+http://search.twitter.com/trends/weekly.json?date=2009-07-10
+http://search.twitter.com/trends/weekly.json?date=2009-07-11
+http://search.twitter.com/trends/weekly.json?date=2009-07-12
+http://search.twitter.com/trends/weekly.json?date=2009-07-13
+http://search.twitter.com/trends/weekly.json?date=2009-07-14
+http://search.twitter.com/trends/weekly.json?date=2009-07-15
+http://search.twitter.com/trends/weekly.json?date=2009-07-16
+http://search.twitter.com/trends/weekly.json?date=2009-07-17
+http://search.twitter.com/trends/daily.json?date=2008-11-19
+http://search.twitter.com/trends/daily.json?date=2008-11-20
+http://search.twitter.com/trends/daily.json?date=2008-11-21
+http://search.twitter.com/trends/daily.json?date=2008-11-22
+http://search.twitter.com/trends/daily.json?date=2008-11-23
+http://search.twitter.com/trends/daily.json?date=2008-11-24
+http://search.twitter.com/trends/daily.json?date=2008-11-25
+http://search.twitter.com/trends/daily.json?date=2008-11-26
+http://search.twitter.com/trends/daily.json?date=2008-11-27
+http://search.twitter.com/trends/daily.json?date=2008-11-28
+http://search.twitter.com/trends/daily.json?date=2008-11-29
+http://search.twitter.com/trends/daily.json?date=2008-11-30
+http://search.twitter.com/trends/daily.json?date=2008-12-01
+http://search.twitter.com/trends/daily.json?date=2008-12-02
+http://search.twitter.com/trends/daily.json?date=2008-12-03
+http://search.twitter.com/trends/daily.json?date=2008-12-04
+http://search.twitter.com/trends/daily.json?date=2008-12-05
+http://search.twitter.com/trends/daily.json?date=2008-12-06
+http://search.twitter.com/trends/daily.json?date=2008-12-07
+http://search.twitter.com/trends/daily.json?date=2008-12-08
+http://search.twitter.com/trends/daily.json?date=2008-12-09
+http://search.twitter.com/trends/daily.json?date=2008-12-10
+http://search.twitter.com/trends/daily.json?date=2008-12-11
+http://search.twitter.com/trends/daily.json?date=2008-12-12
+http://search.twitter.com/trends/daily.json?date=2008-12-13
+http://search.twitter.com/trends/daily.json?date=2008-12-14
+http://search.twitter.com/trends/daily.json?date=2008-12-15
+http://search.twitter.com/trends/daily.json?date=2008-12-16
+http://search.twitter.com/trends/daily.json?date=2008-12-17
+http://search.twitter.com/trends/daily.json?date=2008-12-18
+http://search.twitter.com/trends/daily.json?date=2008-12-19
+http://search.twitter.com/trends/daily.json?date=2008-12-20
+http://search.twitter.com/trends/daily.json?date=2008-12-21
+http://search.twitter.com/trends/daily.json?date=2008-12-22
+http://search.twitter.com/trends/daily.json?date=2008-12-23
+http://search.twitter.com/trends/daily.json?date=2008-12-24
+http://search.twitter.com/trends/daily.json?date=2008-12-25
+http://search.twitter.com/trends/daily.json?date=2008-12-26
+http://search.twitter.com/trends/daily.json?date=2008-12-27
+http://search.twitter.com/trends/daily.json?date=2008-12-28
+http://search.twitter.com/trends/daily.json?date=2008-12-29
+http://search.twitter.com/trends/daily.json?date=2008-12-30
+http://search.twitter.com/trends/daily.json?date=2008-12-31
+http://search.twitter.com/trends/daily.json?date=2009-01-01
+http://search.twitter.com/trends/daily.json?date=2009-01-02
+http://search.twitter.com/trends/daily.json?date=2009-01-03
+http://search.twitter.com/trends/daily.json?date=2009-01-04
+http://search.twitter.com/trends/daily.json?date=2009-01-05
+http://search.twitter.com/trends/daily.json?date=2009-01-06
+http://search.twitter.com/trends/daily.json?date=2009-01-07
+http://search.twitter.com/trends/daily.json?date=2009-01-08
+http://search.twitter.com/trends/daily.json?date=2009-01-09
+http://search.twitter.com/trends/daily.json?date=2009-01-10
+http://search.twitter.com/trends/daily.json?date=2009-01-11
+http://search.twitter.com/trends/daily.json?date=2009-01-12
+http://search.twitter.com/trends/daily.json?date=2009-01-13
+http://search.twitter.com/trends/daily.json?date=2009-01-14
+http://search.twitter.com/trends/daily.json?date=2009-01-15
+http://search.twitter.com/trends/daily.json?date=2009-01-16
+http://search.twitter.com/trends/daily.json?date=2009-01-17
+http://search.twitter.com/trends/daily.json?date=2009-01-18
+http://search.twitter.com/trends/daily.json?date=2009-01-19
+http://search.twitter.com/trends/daily.json?date=2009-01-20
+http://search.twitter.com/trends/daily.json?date=2009-01-21
+http://search.twitter.com/trends/daily.json?date=2009-01-22
+http://search.twitter.com/trends/daily.json?date=2009-01-23
+http://search.twitter.com/trends/daily.json?date=2009-01-24
+http://search.twitter.com/trends/daily.json?date=2009-01-25
+http://search.twitter.com/trends/daily.json?date=2009-01-26
+http://search.twitter.com/trends/daily.json?date=2009-01-27
+http://search.twitter.com/trends/daily.json?date=2009-01-28
+http://search.twitter.com/trends/daily.json?date=2009-01-29
+http://search.twitter.com/trends/daily.json?date=2009-01-30
+http://search.twitter.com/trends/daily.json?date=2009-01-31
+http://search.twitter.com/trends/daily.json?date=2009-02-01
+http://search.twitter.com/trends/daily.json?date=2009-02-02
+http://search.twitter.com/trends/daily.json?date=2009-02-03
+http://search.twitter.com/trends/daily.json?date=2009-02-04
+http://search.twitter.com/trends/daily.json?date=2009-02-05
+http://search.twitter.com/trends/daily.json?date=2009-02-06
+http://search.twitter.com/trends/daily.json?date=2009-02-07
+http://search.twitter.com/trends/daily.json?date=2009-02-08
+http://search.twitter.com/trends/daily.json?date=2009-02-09
+http://search.twitter.com/trends/daily.json?date=2009-02-10
+http://search.twitter.com/trends/daily.json?date=2009-02-11
+http://search.twitter.com/trends/daily.json?date=2009-02-12
+http://search.twitter.com/trends/daily.json?date=2009-02-13
+http://search.twitter.com/trends/daily.json?date=2009-02-14
+http://search.twitter.com/trends/daily.json?date=2009-02-15
+http://search.twitter.com/trends/daily.json?date=2009-02-16
+http://search.twitter.com/trends/daily.json?date=2009-02-17
+http://search.twitter.com/trends/daily.json?date=2009-02-18
+http://search.twitter.com/trends/daily.json?date=2009-02-19
+http://search.twitter.com/trends/daily.json?date=2009-02-20
+http://search.twitter.com/trends/daily.json?date=2009-02-21
+http://search.twitter.com/trends/daily.json?date=2009-02-22
+http://search.twitter.com/trends/daily.json?date=2009-02-23
+http://search.twitter.com/trends/daily.json?date=2009-02-24
+http://search.twitter.com/trends/daily.json?date=2009-02-25
+http://search.twitter.com/trends/daily.json?date=2009-02-26
+http://search.twitter.com/trends/daily.json?date=2009-02-27
+http://search.twitter.com/trends/daily.json?date=2009-02-28
+http://search.twitter.com/trends/daily.json?date=2009-03-01
+http://search.twitter.com/trends/daily.json?date=2009-03-02
+http://search.twitter.com/trends/daily.json?date=2009-03-03
+http://search.twitter.com/trends/daily.json?date=2009-03-04
+http://search.twitter.com/trends/daily.json?date=2009-03-05
+http://search.twitter.com/trends/daily.json?date=2009-03-06
+http://search.twitter.com/trends/daily.json?date=2009-03-07
+http://search.twitter.com/trends/daily.json?date=2009-03-08
+http://search.twitter.com/trends/daily.json?date=2009-03-09
+http://search.twitter.com/trends/daily.json?date=2009-03-10
+http://search.twitter.com/trends/daily.json?date=2009-03-11
+http://search.twitter.com/trends/daily.json?date=2009-03-12
+http://search.twitter.com/trends/daily.json?date=2009-03-13
+http://search.twitter.com/trends/daily.json?date=2009-03-14
+http://search.twitter.com/trends/daily.json?date=2009-03-15
+http://search.twitter.com/trends/daily.json?date=2009-03-16
+http://search.twitter.com/trends/daily.json?date=2009-03-17
+http://search.twitter.com/trends/daily.json?date=2009-03-18
+http://search.twitter.com/trends/daily.json?date=2009-03-19
+http://search.twitter.com/trends/daily.json?date=2009-03-20
+http://search.twitter.com/trends/daily.json?date=2009-03-21
+http://search.twitter.com/trends/daily.json?date=2009-03-22
+http://search.twitter.com/trends/daily.json?date=2009-03-23
+http://search.twitter.com/trends/daily.json?date=2009-03-24
+http://search.twitter.com/trends/daily.json?date=2009-03-25
+http://search.twitter.com/trends/daily.json?date=2009-03-26
+http://search.twitter.com/trends/daily.json?date=2009-03-27
+http://search.twitter.com/trends/daily.json?date=2009-03-28
+http://search.twitter.com/trends/daily.json?date=2009-03-29
+http://search.twitter.com/trends/daily.json?date=2009-03-30
+http://search.twitter.com/trends/daily.json?date=2009-03-31
+http://search.twitter.com/trends/daily.json?date=2009-04-01
+http://search.twitter.com/trends/daily.json?date=2009-04-02
+http://search.twitter.com/trends/daily.json?date=2009-04-03
+http://search.twitter.com/trends/daily.json?date=2009-04-04
+http://search.twitter.com/trends/daily.json?date=2009-04-05
+http://search.twitter.com/trends/daily.json?date=2009-04-06
+http://search.twitter.com/trends/daily.json?date=2009-04-07
+http://search.twitter.com/trends/daily.json?date=2009-04-08
+http://search.twitter.com/trends/daily.json?date=2009-04-09
+http://search.twitter.com/trends/daily.json?date=2009-04-10
+http://search.twitter.com/trends/daily.json?date=2009-04-11
+http://search.twitter.com/trends/daily.json?date=2009-04-12
+http://search.twitter.com/trends/daily.json?date=2009-04-13
+http://search.twitter.com/trends/daily.json?date=2009-04-14
+http://search.twitter.com/trends/daily.json?date=2009-04-15
+http://search.twitter.com/trends/daily.json?date=2009-04-16
+http://search.twitter.com/trends/daily.json?date=2009-04-17
+http://search.twitter.com/trends/daily.json?date=2009-04-18
+http://search.twitter.com/trends/daily.json?date=2009-04-19
+http://search.twitter.com/trends/daily.json?date=2009-04-20
+http://search.twitter.com/trends/daily.json?date=2009-04-21
+http://search.twitter.com/trends/daily.json?date=2009-04-22
+http://search.twitter.com/trends/daily.json?date=2009-04-23
+http://search.twitter.com/trends/daily.json?date=2009-04-24
+http://search.twitter.com/trends/daily.json?date=2009-04-25
+http://search.twitter.com/trends/daily.json?date=2009-04-26
+http://search.twitter.com/trends/daily.json?date=2009-04-27
+http://search.twitter.com/trends/daily.json?date=2009-04-28
+http://search.twitter.com/trends/daily.json?date=2009-04-29
+http://search.twitter.com/trends/daily.json?date=2009-04-30
+http://search.twitter.com/trends/daily.json?date=2009-05-01
+http://search.twitter.com/trends/daily.json?date=2009-05-02
+http://search.twitter.com/trends/daily.json?date=2009-05-03
+http://search.twitter.com/trends/daily.json?date=2009-05-04
+http://search.twitter.com/trends/daily.json?date=2009-05-05
+http://search.twitter.com/trends/daily.json?date=2009-05-06
+http://search.twitter.com/trends/daily.json?date=2009-05-07
+http://search.twitter.com/trends/daily.json?date=2009-05-08
+http://search.twitter.com/trends/daily.json?date=2009-05-09
+http://search.twitter.com/trends/daily.json?date=2009-05-10
+http://search.twitter.com/trends/daily.json?date=2009-05-11
+http://search.twitter.com/trends/daily.json?date=2009-05-12
+http://search.twitter.com/trends/daily.json?date=2009-05-13
+http://search.twitter.com/trends/daily.json?date=2009-05-14
+http://search.twitter.com/trends/daily.json?date=2009-05-15
+http://search.twitter.com/trends/daily.json?date=2009-05-16
+http://search.twitter.com/trends/daily.json?date=2009-05-17
+http://search.twitter.com/trends/daily.json?date=2009-05-18
+http://search.twitter.com/trends/daily.json?date=2009-05-19
+http://search.twitter.com/trends/daily.json?date=2009-05-20
+http://search.twitter.com/trends/daily.json?date=2009-05-21
+http://search.twitter.com/trends/daily.json?date=2009-05-22
+http://search.twitter.com/trends/daily.json?date=2009-05-23
+http://search.twitter.com/trends/daily.json?date=2009-05-24
+http://search.twitter.com/trends/daily.json?date=2009-05-25
+http://search.twitter.com/trends/daily.json?date=2009-05-26
+http://search.twitter.com/trends/daily.json?date=2009-05-27
+http://search.twitter.com/trends/daily.json?date=2009-05-28
+http://search.twitter.com/trends/daily.json?date=2009-05-29
+http://search.twitter.com/trends/daily.json?date=2009-05-30
+http://search.twitter.com/trends/daily.json?date=2009-05-31
+http://search.twitter.com/trends/daily.json?date=2009-06-01
+http://search.twitter.com/trends/daily.json?date=2009-06-02
+http://search.twitter.com/trends/daily.json?date=2009-06-03
+http://search.twitter.com/trends/daily.json?date=2009-06-04
+http://search.twitter.com/trends/daily.json?date=2009-06-05
+http://search.twitter.com/trends/daily.json?date=2009-06-06
+http://search.twitter.com/trends/daily.json?date=2009-06-07
+http://search.twitter.com/trends/daily.json?date=2009-06-08
+http://search.twitter.com/trends/daily.json?date=2009-06-09
+http://search.twitter.com/trends/daily.json?date=2009-06-10
+http://search.twitter.com/trends/daily.json?date=2009-06-11
+http://search.twitter.com/trends/daily.json?date=2009-06-12
+http://search.twitter.com/trends/daily.json?date=2009-06-13
+http://search.twitter.com/trends/daily.json?date=2009-06-14
+http://search.twitter.com/trends/daily.json?date=2009-06-15
+http://search.twitter.com/trends/daily.json?date=2009-06-16
+http://search.twitter.com/trends/daily.json?date=2009-06-17
+http://search.twitter.com/trends/daily.json?date=2009-06-18
+http://search.twitter.com/trends/daily.json?date=2009-06-19
+http://search.twitter.com/trends/daily.json?date=2009-06-20
+http://search.twitter.com/trends/daily.json?date=2009-06-21
+http://search.twitter.com/trends/daily.json?date=2009-06-22
+http://search.twitter.com/trends/daily.json?date=2009-06-23
+http://search.twitter.com/trends/daily.json?date=2009-06-24
+http://search.twitter.com/trends/daily.json?date=2009-06-25
+http://search.twitter.com/trends/daily.json?date=2009-06-26
+http://search.twitter.com/trends/daily.json?date=2009-06-27
+http://search.twitter.com/trends/daily.json?date=2009-06-28
+http://search.twitter.com/trends/daily.json?date=2009-06-29
+http://search.twitter.com/trends/daily.json?date=2009-06-30
+http://search.twitter.com/trends/daily.json?date=2009-07-01
+http://search.twitter.com/trends/daily.json?date=2009-07-02
+http://search.twitter.com/trends/daily.json?date=2009-07-03
+http://search.twitter.com/trends/daily.json?date=2009-07-04
+http://search.twitter.com/trends/daily.json?date=2009-07-05
+http://search.twitter.com/trends/daily.json?date=2009-07-06
+http://search.twitter.com/trends/daily.json?date=2009-07-07
+http://search.twitter.com/trends/daily.json?date=2009-07-08
+http://search.twitter.com/trends/daily.json?date=2009-07-09
+http://search.twitter.com/trends/daily.json?date=2009-07-10
+http://search.twitter.com/trends/daily.json?date=2009-07-11
+http://search.twitter.com/trends/daily.json?date=2009-07-12
+http://search.twitter.com/trends/daily.json?date=2009-07-13
+http://search.twitter.com/trends/daily.json?date=2009-07-14
+http://search.twitter.com/trends/daily.json?date=2009-07-15
+http://search.twitter.com/trends/daily.json?date=2009-07-16
+http://search.twitter.com/trends/daily.json?date=2009-07-17
View
16 lib/wuclan/domains/twitter/scrape/session.rb → ...ains/twitter/scrape/twitter_search_job.rb
@@ -9,7 +9,7 @@ module Scrape
# previous scrape session.
#
#
- class Session < Struct.new(
+ class TwitterSearchJob < Struct.new(
:query_term,
:priority,
:prev_rate, :prev_items, :prev_span_min, :prev_span_max
@@ -33,18 +33,18 @@ def initialize *args
# Generate paginated TwitterSearchScrapeRequest
#
def make_request page, pageinfo
- url_str = "http://search.twitter.com/search.json?q=#{query_term}&rpp=#{items_per_page}"
+ url_str = base_url
+ url_str << "&rpp=#{items_per_page}"
url_str << "&max_id=#{unscraped_span.max-1}" if unscraped_span.max
Wuclan::Domains::Twitter::Scrape::TwitterSearchRequest.new url_str
end
- # def initialize query_term, num_items=nil, min_span=nil, max_span=nil, min_timespan=nil, max_timespan=nil
- # self.num_items = num_items.to_i
- # self.prev_span = UnionInterval.new(min_span.to_i, max_span.to_i) if min_span || max_span
- # self.prev_timespan = UnionInterval.new(Time.parse(min_timespan), Time.parse(max_timespan)) rescue nil
- # super(query_term)
- # end
#
+ # Durable handle for this resource, independent of the page/max_id/whatever
+ #
+ def base_url
+ "http://search.twitter.com/search.json?q=#{query_term}"
+ end
end
end
end
View
0 lib/wuclan/domains/twitter/scrape/request.rb → .../twitter/scrape/twitter_search_request.rb
File renamed without changes.
View
2 ripd
View
2 twitter_search_load_jobs.rb
@@ -29,6 +29,8 @@
# Log every N requests
periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iter_interval => 100, :time_interval => 10)
+# # Persist session jobs in distributed DB
+# store = Monkeyshines::ScrapeStore::ReadThruStore.new_from_command_line opts
Twitter::Scrape::Session.hard_request_limit = 5
View
31 twitter_search_scrape.rb
@@ -13,29 +13,31 @@
# Command line options
#
opts = Trollop::options do
- opt :dumpfile_dir, "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
- opt :dumpfile_pattern, "Pattern for dump file output", :default => ":dumpfile_dir/:handle_prefix/:handle/:date/:handle+:datetime-:pid.tsv"
- opt :dumpfile_chunk_time, "Time in seconds to keep dump files open", :default => 60*60*4, :type => Integer
- opt :handle, "Handle to uniquely identify this scrape", :default => 'com.twitter.search'
+ opt :dumpfile_dir, "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
+ opt :dumpfile_pattern, "Pattern for dump file output",
+ :default => ":dumpfile_dir/:handle_prefix/:handle/:date/:handle+:datetime-:pid.tsv"
+ opt :dumpfile_chunk_time, "Frequency to rotate chunk files (in seconds)", :type => Integer,
+ :default => 60*60*4
+ opt :handle, "Handle to uniquely identify this scrape",
+ :default => 'com.twitter.search'
+ opt :items_per_session, "Desired item count per session",
+ :default => 980
+ opt :min_resched_delay, "Don't run jobs more often than this (in seconds)",
+ :default => 60*1
end
Trollop::die :dumpfile_dir unless opts[:dumpfile_dir]
# Queue of request sessions, with reschedule requests
-request_queue = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::Session, RESCHEDULE_GOAL)
+beanstalk_tube = opts[:handle].gsub(/\w+/,'_')
+request_queue = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::Session, opts[:items_per_session], opts.slice(:min_resched_delay))
# Scrape Store for completed requests
dumpfile_pattern = Monkeyshines::Utils::FilenamePattern.new(opts[:dumpfile_pattern], opts.slice(:handle, :dumpfile_dir))
store = Monkeyshines::ScrapeStore::ChunkedFlatFileStore.new dumpfile_pattern, opts[:dumpfile_chunk_time].to_i
# Scrape requests by HTTP
scraper = Monkeyshines::ScrapeEngine::HttpScraper.new Monkeyshines::CONFIG[:twitter]
-# Log every N requests
+# Log every 60 seconds
periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:time_interval => 60)
-
-RESCHEDULE_GOAL = 650
-Twitter::Scrape::Session.hard_request_limit = 15
-request_queue.min_resched_delay = 180
-
-
request_queue.each do |session|
# Run through all pages for this search term
session.each_request do |req|
@@ -48,7 +50,8 @@
# return it to the session for bookkeeping
response
end
+ sleep 0.5
end
request_queue.finish
-#
-# # # (1..50).map{ begin j = bs.reserve(1) ; rescue Exception => e ; warn e ; break ; end ; if j then q = j.body.gsub(/\t.*/,"") ; queries[q] ||= j.id ; if (queries[q] != j.id) then j.delete end ; j.release 65536, 45 ; puts q ; q end rescue 'error' }
+
+# Twitter::Scrape::Session.hard_request_limit = 15
View
41 twitter_trending_scrape.rb
@@ -6,6 +6,7 @@
require 'wukong'
require 'monkeyshines'
require 'wuclan/domains/twitter/scrape' ; include Wuclan::Domains
+
require 'monkeyshines/scrape_engine/http_scraper'
require 'monkeyshines/utils/filename_pattern'
@@ -13,27 +14,45 @@
# Command line options
#
opts = Trollop::options do
- opt :dumpfile_filename, "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
- opt :dumpfile_pattern, "Pattern for dump file output", :default => ":dumpfile_dir/:handle_prefix/:handle/:date/:handle+:datetime-:pid.tsv"
- opt :dumpfile_chunk_time, "Time in seconds to keep dump files open", :default => 60*60*4, :type => Integer
- opt :handle, "Handle to uniquely identify this scrape", :default => 'com.twitter.search'
+ opt :dumpfile_dir, "Filename base to store output. e.g. --dump_basename=/data/ripd", :type => String
+ opt :dumpfile_pattern, "Pattern for dump file output",
+ :default => ":dumpfile_dir/:handle_prefix/:handle/:date/:handle+:datetime-:pid.tsv"
+ opt :dumpfile_chunk_time, "Frequency to rotate chunk files (in seconds)", :type => Integer,
+ :default => 60*60*24
+ opt :handle, "Handle to uniquely identify this scrape",
+ :default => 'com.twitter.search'
+ opt :min_resched_delay, "Don't run jobs more often than this (in seconds)",
+ :default => 60*1
end
Trollop::die :dumpfile_dir unless opts[:dumpfile_dir]
# Queue of request sessions, with reschedule requests
-request_queue = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::Session, RESCHEDULE_GOAL)
+# opts[:beanstalk_tube] ||= opts[:handle].gsub(/\w+/,'_')
+request_queue = Monkeyshines::RequestStream::BeanstalkQueue.new(nil, Twitter::Scrape::Session, opts[:items_per_session], opts.slice(:min_resched_delay)) # , :beanstalk_tube
# Scrape Store for completed requests
dumpfile_pattern = Monkeyshines::Utils::FilenamePattern.new(opts[:dumpfile_pattern], opts.slice(:handle, :dumpfile_dir))
store = Monkeyshines::ScrapeStore::ChunkedFlatFileStore.new dumpfile_pattern, opts[:dumpfile_chunk_time].to_i
# Scrape requests by HTTP
scraper = Monkeyshines::ScrapeEngine::HttpScraper.new Monkeyshines::CONFIG[:twitter]
-# Log every N requests
+# Log every 60 seconds
periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:time_interval => 60)
-RESCHEDULE_GOAL = 650
-Twitter::Scrape::Session.hard_request_limit = 15
-request_queue.min_resched_delay = 180
+class Session < Struct.new(
+ :query_term,
+ :priority,
+ :period
+ )
+
+end
+
+
+# %w[
+# http://search.twitter.com/trends/current.format , 60*60
+# http://search.twitter.com/trends/daily.json?date=2009-03-19
+# http://search.twitter.com/trends/weekly.json?date=2009-03-19
+# ]
+
request_queue.each do |session|
@@ -50,5 +69,5 @@
end
end
request_queue.finish
-#
-# # # (1..50).map{ begin j = bs.reserve(1) ; rescue Exception => e ; warn e ; break ; end ; if j then q = j.body.gsub(/\t.*/,"") ; queries[q] ||= j.id ; if (queries[q] != j.id) then j.delete end ; j.release 65536, 45 ; puts q ; q end rescue 'error' }
+
+# Twitter::Scrape::Session.hard_request_limit = 15

0 comments on commit 328e1d6

Please sign in to comment.
Something went wrong with that request. Please try again.