Merge branch 'master' of github.com:sunlightlabs/openstates

JoeGermuska · Mar 6, 2013 · c8e14c5 · c8e14c5
2 parents 09b401e + 941d21f
commit c8e14c5
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 11 deletions.
diff --git a/openstates/ga/bills.py b/openstates/ga/bills.py
@@ -105,6 +105,7 @@ def scrape(self, session, chambers):
                 "SRA": ["bill:passed"],
                 "HPA": ["bill:passed"],
                 "SPA": ["bill:passed"],
+                "HTABL": ["other"],  # "House Tabled" - what is this?
                 "HCFR": ["committee:passed:favorable"],
                 "SCFR": ["committee:passed:favorable"],
                 "HRAR": ["committee:referred"],
@@ -113,10 +114,11 @@ def scrape(self, session, chambers):
                 "SAHAS": ["other"],
                 "SE": ["bill:passed"],
                 "SR": ["bill:passed"],
-                # STP \m/
                 "HTR": ["bill:reading:3"],
                 "HASAS": ["other"],
                 "SAPPT": ["other"],
+                "HCA": ["other"],
+                "SCRA": ["other"],
                 "S2R": ["bill:reading:2"],
                 "H2R": ["bill:reading:2"],
                 "SENG": ["bill:passed"],
@@ -130,8 +132,7 @@ def scrape(self, session, chambers):
                 "HSG": ["governor:received"],
                 "EFF": ["other"],
                 "STH": ["other"],
-                "HTS": ["other"],  # House transmitted to senate
-                #"": [],
+                "HTS": ["other"],
             }
 
             for action in actions:

diff --git a/openstates/ia/bills.py b/openstates/ia/bills.py
@@ -72,9 +72,6 @@ def scrape(self, chamber, session):
             if bill_id == 'Pick One':
                 continue
 
-            if 'HSB' in bill_id or 'SSB' in bill_id:
-                continue
-
             bill_url = option.attrib['value'].strip() + '&frm=2'
 
             self.scrape_bill(chamber, session, bill_id, bill_url)
@@ -97,6 +94,9 @@ def scrape_bill(self, chamber, session, bill_id, url):
             self.warning("URL: %s gives us an *EMPTY* bill. Aborting." % url)
             return
 
+        if title.lower().startswith("in"):
+            title = page.xpath("string(//table[2]/tr[3])").strip()
+
         if 'HR' in bill_id or 'SR' in bill_id:
             bill_type = ['resolution']
         elif 'HJR' in bill_id or 'SJR' in bill_id:
@@ -149,7 +149,7 @@ def scrape_bill(self, chamber, session, bill_id, url):
             bill.add_sponsor('primary', sponsor)
 
         for tr in page.xpath("//table[3]/tr"):
-            date = tr.xpath("string(td[1])").strip()
+            date = tr.xpath("string(td[contains(text(), ', 20')])").strip()
             if date.startswith("***"):
                 continue
             elif "No history is recorded at this time." in date:
@@ -176,6 +176,8 @@ def scrape_bill(self, chamber, session, bill_id, url):
                 actor = 'upper'
             elif 'H.J.' in action or 'HCS' in action:
                 actor = 'lower'
+            else:
+                actor = "other"
 
             action = re.sub(r'(H|S)\.J\.\s+\d+\.$', '', action).strip()
 

diff --git a/openstates/la/bills.py b/openstates/la/bills.py
@@ -49,6 +49,9 @@ def bill_pages(self, bill_type):
             )
             yield page
 
+    def scrape_bare_page(self, url):
+        page = self.lxmlize(url)
+        return page.xpath("//a")
 
     def scrape(self, chamber, session):
         for bill_type in bill_types[chamber]:
@@ -71,4 +74,17 @@ def scrape_bill_page(self, chamber, bill_url):
             page,
             "//a[@id='ctl00_PageBody_LinkAuthor']/text()"
         )
-        print author
+
+        sbp = lambda x: self.scrape_bare_page(page.xpath(
+            "//a[contains(text(), '%s')]" % (x))[0].attrib['href'])
+
+        authors = [x.text for x in sbp("Authors")]
+        digests = sbp("Digests")
+        versions = sbp("Text")
+
+        title = page.xpath(
+            "//span[@id='ctl00_PageBody_LabelShortTitle']/text()")[0]
+        actions = page.xpath(
+            "//div[@id='ctl00_PageBody_PanelBillInfo']/"
+            "/table[@style='font-size:small']/tr")
+        print actions
diff --git a/openstates/mn/bills.py b/openstates/mn/bills.py
@@ -222,9 +222,9 @@ def scrape(self, chamber, session):
 
         self.get_bill_topics(chamber, session)
 
-        # MN bill search page returns a maximum of 999 search results
+        # MN bill search page returns a maximum 500 search results
         total_rows = list() # used to concatenate search results
-        stride = 900
+        stride = 500
         start = 0
 
         # get total list of rows

diff --git a/openstates/mo/committees.py b/openstates/mo/committees.py
@@ -67,7 +67,7 @@ def scrape_reps_committees(self, term_name, chamber):
         url = '{base}ActiveCommittees.aspx'.format(base=self.reps_url_base)
         page_string = self.urlopen(url)
         page = lxml.html.fromstring(page_string)
-        table = page.xpath('id("contentdata")/table[1]')[0]
+        table = page.xpath('//div[@class="lightened"]/table[1]')[0]
         # Last tr has the date
         trs = table.xpath('tr')[:-1]
         for tr in trs: