Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 99 lines (86 sloc) 3.723 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
import datetime as dt
from dbfpy import dbf
import pytz
import re
from .utils import DBFMixin

from billy.scrape.events import EventScraper, Event

agenda_dbf = "ftp://www.njleg.state.nj.us/ag/2012data/AGENDAS.DBF"

class NJEventScraper(EventScraper, DBFMixin):
    state = 'nj'
    _tz = pytz.timezone('US/Eastern')

    def initialize_committees(self, year_abr):
        chamber = {'A':'Assembly', 'S': 'Senate', '':''}

        com_url, com_db = self.get_dbf(year_abr, 'COMMITT')

        self._committees = {}
        # There are some IDs that are missing. I'm going to add them
        # before we load the DBF, in case they include them, we'll just
        # override with their data.
        #
        # This data is from:
        # http://www.njleg.state.nj.us/media/archive_audio2.asp?KEY=<KEY>&SESSION=2012
        overlay = {
            'A': 'Assembly on the Whole',
            'S': 'Senate on the Whole',
            'J': 'Joint Legislature on the Whole',
            'ABUB': 'Assembly Budget Committee',
            'JBOC': 'Joint Budget Oversight',
            'JPS': 'Joint Committee on the Public Schools',
            'LRC': 'New Jersey Law Revision Commission',
            'PHBC': 'Pension and Health Benefits Review Commission',
            'SBAB': 'Senate Budget and Appropriations Committee',
            'SUTC': 'Sales and Use Tax Review Commission',
            'SPLS': 'Special Session'
        }
        self._committees = overlay

        for com in com_db:
            # map XYZ -> "Assembly/Senate _________ Committee"
            self._committees[com['CODE']] = ' '.join((chamber[com['HOUSE']],
                                                      com['DESCRIPTIO'],
                                                      'Committee'))

    def scrape(self, chamber, session):
        year_abr = ((int(session) - 209) * 2) + 2000
        self.initialize_committees(year_abr)
        url, db = self.get_dbf(year_abr, "AGENDAS")
        records = [ x.asDict() for x in db ]
        for record in records:
            if record['STATUS'] != "Scheduled":
                continue
            description = record['COMMENTS']
            related_bills = []

            for bill in re.findall("(A|S)(-)?(\d{4})", description):
                related_bills.append({
                    "bill_id" : "%s %s" % ( bill[0], bill[2] ),
                    "descr": description
                })

            date_time = "%s %s" % (
                record['DATE'],
                record['TIME']
            )
            date_time = dt.datetime.strptime(date_time, "%m/%d/%Y %I:%M %p")
            hr_name = self._committees[record['COMMHOUSE']]

            event = Event(
                session,
                date_time,
                'committee:meeting',
                "Meeting of the %s" % ( hr_name ),
                location=record['LOCATION'] or "Statehouse",
            )
            for bill in related_bills:
                event.add_related_bill(bill['bill_id'],
                                      description=bill['descr'],
                                      type='consideration')
            try:
                chamber = {
                    "a" : "lower",
                    "s" : "upper",
                    "j" : "joint"
                }[record['COMMHOUSE'][0].lower()]
            except KeyError:
                chamber = "joint"

            event.add_participant("host",
                                  hr_name,
                                  'committee',
                                  committee_code=record['COMMHOUSE'],
                                  chamber=chamber)
            event.add_source(agenda_dbf)
            self.save_event(event)
Something went wrong with that request. Please try again.