Permalink
Browse files

added q'n'd replays scrapers

  • Loading branch information...
1 parent b098ba1 commit e0493790a0b1b7ce6e83bd7240e1443f16e39fbb @syhw committed Jun 28, 2011
Showing with 34 additions and 0 deletions.
  1. +6 −0 scrapers/GG_rep_dler.sh
  2. +28 −0 scrapers/TL_rep_dler.py
View
@@ -0,0 +1,6 @@
+#!/bin/bash
+for ((i = 0; i <= 45648; i++));
+do
+ wget http://www.gosugamers.net/starcraft/admin/a_replays.php?dl=$i
+done
+rm index.html*
View
@@ -0,0 +1,28 @@
+#!/opt/local/bin/python2.6
+
+# http://www.teamliquid.net/replay/
+# http://www.gosugamers.net/starcraft/replays/
+import urllib2
+
+#TEST showing that the redirect works
+#url = 'http://www.teamliquid.net/replay/download.php?replay=1801'
+#url = 'http://www.gosugamers.net/starcraft/admin/a_replays.php?dl=30084'
+#page = urllib2.urlopen(url)
+#print page.geturl()
+
+url = 'http://www.teamliquid.net/replay/download.php?replay='
+for i in range(1964):
+ try:
+ page = urllib2.urlopen(url+str(i+1))
+ except:
+ continue
+ repnamel = page.geturl().split('/')
+ repname = repnamel[len(repnamel)-1]
+ file = open('teamliquid/'+repname,"wb")
+ file.write(page.read())
+ print "saved replay ",
+ print repname
+ file.close()
+
+
+

0 comments on commit e049379

Please sign in to comment.