@@ -26,4 +26,5 @@ sgf/titles/*.sgf
svn/*
titles/titles.html
titles/titles.db
-
+titles/news.html
+titles/lg.html
\ No newline at end of file
.gitignore
@@ -26,50 +26,71 @@ cur.executescript("""
""")
base_url = "http://igo-kisen.hp.infoseek.co.jp/"
-# base_url = "http://localhost/eidogo/titles/"
-page = urllib2.urlopen(base_url + "topics.html")
-soup = BeautifulSoup(page)
-
-trs = soup.find("table", {"width": "1050"}).findAll("tr")
-dt = ""
-
+# base_url = "http://eidogo_dev/titles/"
re_tags = re.compile("<[^>]+>")
-scraped_subpages = []
-
-for tr in trs:
- tds = tr.findAll("td")
- if (len(tds) == 0):
- continue
+def scrape_news():
+ page = urllib2.urlopen(base_url + "news.html")
+ soup = BeautifulSoup(page)
+ scraped_subpages = []
+ trs = soup.findAll("tr", {"align": "center"})
+ for tr in trs:
+ tds = tr.findAll("td")
+ if (len(tds) == 0):
+ continue
+ sig = []
+ for td in tds:
+ sig.append(re_tags.sub("", str(td.contents[0])))
+ sig = ' '.join(sig)
+
+ cur.execute("select * from notices where sig=?", (sig,))
+ if (cur.fetchone()):
+ continue
+ cur.execute("insert into notices (sig) values (?)", (sig,))
+
+ subpage_fn = tr.a['href']
+ if (scraped_subpages.count(subpage_fn) > 0):
+ continue
+ print subpage_fn
+ scraped_subpages.append(subpage_fn)
+ scrape_subpage(subpage_fn)
- td0 = tds.pop(0).contents[0]
- if (td0 != " "):
- dt = td0
+def scrape_topics():
+ page = urllib2.urlopen(base_url + "topics.html")
+ soup = BeautifulSoup(page)
+ scraped_subpages = []
+ trs = soup.find("table", {"width": "1050"}).findAll("tr")
+ dt = ""
+ for tr in trs:
+ tds = tr.findAll("td")
+ if (len(tds) == 0):
+ continue
+ td0 = tds.pop(0).contents[0]
+ if (td0 != " "):
+ dt = td0
+ sig = [dt]
+ for td in tds:
+ sig.append(re_tags.sub("", str(td.contents[0])))
+ sig = ' '.join(sig)
- sig = [dt]
- for td in tds:
- sig.append(re_tags.sub("", str(td.contents[0])))
-
- sig = ' '.join(sig)
-
- cur.execute("select * from notices where sig=?", (sig,))
- if (cur.fetchone()):
- continue
-
- cur.execute("insert into notices (sig) values (?)", (sig,))
-
- subpage_fn = tr.a['href']
-
- if (scraped_subpages.count(subpage_fn) > 0):
- continue
+ cur.execute("select * from notices where sig=?", (sig,))
+ if (cur.fetchone()):
+ continue
+ cur.execute("insert into notices (sig) values (?)", (sig,))
- print subpage_fn
- scraped_subpages.append(subpage_fn)
+ subpage_fn = tr.a['href']
+ if (scraped_subpages.count(subpage_fn) > 0):
+ continue
+ print subpage_fn
+ scraped_subpages.append(subpage_fn)
+ scrape_subpage(subpage_fn)
+
+def scrape_subpage(subpage_fn):
try:
subpage = urllib2.urlopen(base_url + subpage_fn)
except:
print "! not found"
- continue
+ return
subsoup = BeautifulSoup(subpage)
sgf_path = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../sgf/titles'))
@@ -90,27 +111,39 @@ for tr in trs:
print " " + fn
sgf = sgfparser.Cursor(raw_sgf)
info = sgf.getRootNode(0)
- cur.execute("insert into games (fn, dt, ev, pw, pb, re) values (?,?,?,?,?,?)",
- (fn, info['DT'][0], info['EV'][0], info['PW'][0] + ' ' + info['WR'][0], info['PB'][0] + ' ' + info['BR'][0], info['RE'][0]))
+ cur.execute(
+ "insert into games (fn, dt, ev, pw, pb, re) values (?,?,?,?,?,?)",
+ (fn,
+ info['DT'][0],
+ info['EV'][0],
+ info['PW'][0] + ' ' + info['WR'][0],
+ info['PB'][0] + ' ' + info['BR'][0],
+ info['RE'][0]))
f = open(os.path.join(sgf_path, fn), "w")
f.write(raw_sgf)
f.close()
-cur.execute("select * from games order by dt desc limit 250")
-rows = cur.fetchall()
+def output_games():
+ cur.execute("select * from games order by dt desc limit 250")
+ rows = cur.fetchall()
+
+ f = open(os.path.join(curdir, 'titles.html'), "w")
+ f.write("<table id='tourney-games'><tr>" +
+ "<th>Date</th><th>Event</th><th>White</th><th>Black</th><th>Result</th>" +
+ "</tr>")
+ cl = ""
+ for row in rows:
+ if (cl == " class='odd'"):
+ cl = " class='even'"
+ else:
+ cl = " class='odd'"
+ fn = row[0].replace(".sgf", "")
+ f.write("<tr" + cl + ">")
+ for col in row[1:]:
+ f.write("<td><a href='./#titles/" + fn + "'>" + col + "</a></td>")
+ f.write("</tr>")
+ f.write("</table>")
+ f.close()
-f = open(os.path.join(curdir, 'titles.html'), "w")
-f.write("<table id='tourney-games'><tr><th>Date</th><th>Event</th><th>White</th><th>Black</th><th>Result</th></tr>")
-cl = ""
-for row in rows:
- if (cl == " class='odd'"):
- cl = " class='even'"
- else:
- cl = " class='odd'"
- fn = row[0].replace(".sgf", "")
- f.write("<tr" + cl + ">")
- for col in row[1:]:
- f.write("<td><a href='./#titles/" + fn + "'>" + col + "</a></td>")
- f.write("</tr>")
-f.write("</table>")
-f.close()
\ No newline at end of file
+scrape_news()
+output_games()
\ No newline at end of file
titles/scrape.py
f3486f765f327bd39818dafa0520072d605ea377
Justin Kramer
jkkramer@gmail.com
http://github.com/jkk/eidogo/commit/7748bb95fdaa161430967d8640a026e93995ef22
7748bb95fdaa161430967d8640a026e93995ef22
2008-04-30T21:12:39-07:00
2008-04-30T21:12:39-07:00
updated titles scraper
2693803c0df2f81653e5d8a7cfc1ea5baba0aad6
Justin Kramer
jkkramer@gmail.com