move past errors when generating featured

datawrangling · Jun 21, 2009 · 43aa6cb · 43aa6cb
1 parent 64571ef
commit 43aa6cb
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 10 deletions.
diff --git a/lib/scripts/daily_load.sh b/lib/scripts/daily_load.sh
@@ -62,7 +62,7 @@ cd /mnt
 python /mnt/app/current/lib/scripts/generate_featured_pages.py -d $MAXDATE > /mnt/featured_pages.txt
 time mysql -u root trendingtopics_production <  /mnt/app/current/lib/sql/load_featured_pages.sql
 
-time mysql -u root trendingtopics_production -e "update pages p set p.featured = (select 1 from new_featured_pages fp where fp.page_id=p.id);"
+time mysql -u root trendingtopics_production -e "update new_pages p set p.featured = (select 1 from new_featured_pages fp where fp.page_id=p.id);"
 
 echo archiving the data to S3
 # back up the trendsdb data, this copy will be pulled by the next daily job

diff --git a/lib/scripts/generate_featured_pages.py b/lib/scripts/generate_featured_pages.py
@@ -70,17 +70,23 @@ def featured_pages(date):
   base = 'http://en.wikipedia.org/wiki/Wikipedia:Today%27s_featured_article/' 
   # get previous 3 days of featured articles...
   url = base + date.strftime("%B_%d,_%Y")
-  soup = soupify_url(url)
-  div = soup.findAll(id="bodyContent")
-  titles = get_titles(div[0])
+  try:
+    soup = soupify_url(url)
+    div = soup.findAll(id="bodyContent")
+    titles = get_titles(div[0])
+  except:
+    titles = []  
   return titles
 
 def featured_pictures(date):
   base = 'http://en.wikipedia.org/wiki/Template:POTD/'
   url = base + date.strftime("%Y-%m-%d")
-  soup = soupify_url(url)
-  table = soup.findAll(cellspacing="5")
-  titles = get_titles(table[0])  
+  try:
+    soup = soupify_url(url)
+    table = soup.findAll(cellspacing="5")
+    titles = get_titles(table[0])  
+  except:
+    titles = []    
   return titles
 
 def date_pages(date):
@@ -89,9 +95,12 @@ def date_pages(date):
 def anniversaries(date):
   base = 'http://en.wikipedia.org/wiki/Wikipedia:Selected_anniversaries/'
   url = base + date.strftime("%B_%d")
-  soup = soupify_url(url)
-  div = soup.findAll(id="bodyContent")
-  titles = get_titles(div[0])
+  try:
+    soup = soupify_url(url)
+    div = soup.findAll(id="bodyContent")
+    titles = get_titles(div[0])
+  except:
+    titles = []    
   return titles  
 
 def titles_for_date(date):