flake8 everything but tests

.pep8

@@ -0,0 +1,4 @@
+[pep8]
+ignore=E124
+# ignore warning about alignment of last brace, it is nicer to have it
+# matching the first brace than indented an extra level

billy/bin/update.py

@@ -38,7 +38,7 @@ def _get_configured_scraper(scraper_type, options, metadata):
     except ScrapeError as e:
         # silence error only when alldata is present
         if ('alldata' in options.types and
-            str(e.orig_exception) == 'No module named %s' % scraper_type):
+                str(e.orig_exception) == 'No module named %s' % scraper_type):
             return None
         else:
             raise e
@@ -159,14 +159,14 @@ def _do_imports(abbrev, args):
 
     if 'legislators' in args.types:
         report['legislators'] = \
-                import_legislators(abbrev, settings.BILLY_DATA_DIR)
+            import_legislators(abbrev, settings.BILLY_DATA_DIR)
 
     if 'bills' in args.types:
         report['bills'] = import_bills(abbrev, settings.BILLY_DATA_DIR)
 
     if 'committees' in args.types:
         report['committees'] = \
-                import_committees(abbrev, settings.BILLY_DATA_DIR)
+            import_committees(abbrev, settings.BILLY_DATA_DIR)
 
     if 'events' in args.types or 'speeches' in args.types:
         report['events'] = import_events(abbrev, settings.BILLY_DATA_DIR)
@@ -205,21 +205,21 @@ def _do_reports(abbrev, args):
 def main():
     try:
         parser = argparse.ArgumentParser(
-          description='update billy data',
-          parents=[base_arg_parser],
+            description='update billy data',
+            parents=[base_arg_parser],
         )
 
-        what = parser.add_argument_group('what to scrape',
-                                 'flags that help select what data to scrape')
+        what = parser.add_argument_group(
+            'what to scrape', 'flags that help select what data to scrape')
         scrape = parser.add_argument_group('scraper config',
-                                 'settings for the scraper')
+                                           'settings for the scraper')
 
         parser.add_argument('module', type=str, help='scraper module (eg. nc)')
         what.add_argument('-s', '--session', action='append',
-                            dest='sessions', default=[],
+                          dest='sessions', default=[],
                           help='session(s) to scrape')
         what.add_argument('-t', '--term', action='append', dest='terms',
-                            help='term(s) to scrape', default=[])
+                          help='term(s) to scrape', default=[])
 
         for arg in ('upper', 'lower'):
             what.add_argument('--' + arg, action='append_const',

billy/commands/download_photos.py

@@ -22,7 +22,7 @@ def _upload(fname, bucket):
     headers = {'Cache-Control': 'max-age=2592000'}
 
     # optimize JPEG
-    subprocess.check_call(['jpegoptim',  '--strip-all', fname])
+    subprocess.check_call(['jpegoptim', '--strip-all', fname])
 
     k = Key(bucket)
     k.key = fname

billy/commands/dump.py

@@ -90,7 +90,7 @@ def dump_legislator_csvs(abbr):
     leg_fields = ('leg_id', 'full_name', 'first_name', 'middle_name',
                   'last_name', 'suffixes', 'nickname', 'active',
                   settings.LEVEL_FIELD, 'chamber', 'district', 'party',
-                   'transparencydata_id', 'photo_url', 'created_at',
+                  'transparencydata_id', 'photo_url', 'created_at',
                   'updated_at')
     leg_csv_fname, leg_csv = _make_csv(abbr, 'legislators.csv', leg_fields)
 
@@ -155,8 +155,8 @@ def dump_bill_csvs(abbr):
     for bill in db.bills.find({settings.LEVEL_FIELD: abbr}):
         bill_csv.writerow(extract_fields(bill, bill_fields))
 
-        bill_info = extract_fields(bill,
-                   ('bill_id', settings.LEVEL_FIELD, 'session', 'chamber'))
+        bill_info = extract_fields(
+            bill, ('bill_id', settings.LEVEL_FIELD, 'session', 'chamber'))
         _bill_info[bill['_id']] = bill_info
 
         # basically same behavior for actions, sponsors and votes:
@@ -181,9 +181,9 @@ def dump_bill_csvs(abbr):
         for vtype in ('yes', 'no', 'other'):
             for leg_vote in vote[vtype + '_votes']:
                 legvote_csv.writerow({'vote_id': vote['vote_id'],
-                                  'leg_id': leg_vote['leg_id'],
-                                  'name': leg_vote['name'].encode('utf8'),
-                                  'vote': vtype})
+                                      'leg_id': leg_vote['leg_id'],
+                                      'name': leg_vote['name'].encode('utf8'),
+                                      'vote': vtype})
 
     return (bill_csv_fname, action_csv_fname, sponsor_csv_fname,
             vote_csv_fname, legvote_csv_fname)
@@ -195,11 +195,12 @@ class DumpCSV(BaseCommand):
 
     def add_args(self):
         self.add_argument('abbrs', metavar='ABBR', type=str, nargs='+',
-                  help='the two-letter abbreviation for the data to export')
+                          help='the abbreviation for the data to export')
         self.add_argument('--file', '-f',
-                  help='filename to output to (defaults to <abbr>.zip)')
+                          help='filename to output to (defaults to <abbr>.zip)'
+                         )
         self.add_argument('--upload', '-u', action='store_true', default=False,
-                  help='upload the created archives to S3')
+                          help='upload the created archives to S3')
 
     def handle(self, args):
         for abbr in args.abbrs:
@@ -227,17 +228,18 @@ class DumpJSON(BaseCommand):
 
     def add_args(self):
         self.add_argument('abbrs', metavar='ABBR', type=str, nargs='+',
-                  help='the two-letter abbreviation for the data to export')
+                          help='the abbreviation for the data to export')
         self.add_argument('--file', '-f',
-                  help='filename to output to (defaults to <abbr>.zip)')
+                          help='filename to output to (defaults to <abbr>.zip)'
+                         )
         self.add_argument('--upload', '-u', action='store_true', default=False,
-                  help='upload the created archives to S3')
+                          help='upload the created archives to S3')
         self.add_argument('--apikey', dest='API_KEY',
-                  help='the API key to use')
+                          help='the API key to use')
         self.add_argument('--schema_dir', default=None,
-                  help='directory to use for API schemas (optional)')
+                          help='directory to use for API schemas (optional)')
         self.add_argument('--novalidate', action='store_true', default=False,
-                  help="don't run validation")
+                          help="don't run validation")
 
     def handle(self, args):
         for abbr in args.abbrs:

billy/commands/sync_versions.py

@@ -30,10 +30,9 @@ def handle(self, args):
         documents = db.tracked_versions.find(spec, timeout=False)
         doc_count = documents.count()
 
-        print 'starting {0} for {1} documents ({2})'.format(task.__name__,
-                                                            doc_count,
-                                    'immediate' if args.immediate else 'queued'
-                                                           )
+        print 'starting {0} for {1} documents ({2})'.format(
+            task.__name__, doc_count,
+            'immediate' if args.immediate else 'queued')
 
         for doc in documents:
             if args.immediate:

billy/commands/textextract.py

@@ -23,8 +23,8 @@ def handle(self, args):
         extract_text = module.extract_text
 
         if (args.filename.startswith('http://') or
-            args.filename.startswith('https://') or
-            args.filename.startswith('ftp://')):
+                args.filename.startswith('https://') or
+                args.filename.startswith('ftp://')):
             filedata = urllib2.urlopen(args.filename).read()
         else:
             filedata = open(args.filename).read()

billy/commands/update_external_ids.py

@@ -46,25 +46,27 @@ def handle(self, args):
             abbrev = meta['_id'].upper()
 
             for leg in db.legislators.find(query):
-                query = urllib.urlencode({'apikey': settings.API_KEY,
-                                  'search': leg['full_name'].encode('utf8')})
+                query = urllib.urlencode({
+                    'apikey': settings.API_KEY,
+                    'search': leg['full_name'].encode('utf8')
+                })
                 url = ('http://transparencydata.com/api/1.0/entities.json?' +
                        query)
                 data = urllib2.urlopen(url).read()
                 results = json.loads(data)
                 matches = []
                 for result in results:
                     if (result['state'] == abbrev and
-                        result['seat'][6:] == leg['chamber'] and
-                        result['type'] == 'politician'):
+                            result['seat'][6:] == leg['chamber'] and
+                            result['type'] == 'politician'):
                         matches.append(result)
 
                 if len(matches) == 1:
                     leg['transparencydata_id'] = matches[0]['id']
                     db.legislators.save(leg, safe=True)
                     updated += 1
 
-            print 'Updated %s of %s missing transparencydata ids' % (updated,
-                                                             initial_count)
+            print 'Updated %s of %s missing transparencydata ids' % (
+                updated, initial_count)
 
             time.sleep(30)

billy/commands/validate_api.py

@@ -17,10 +17,10 @@ class ValidateApi(BaseCommand):
     help = 'validate data from the API'
 
     def add_args(self):
-        self.add_argument('--apikey', dest='API_KEY',
-                  help='the API key to use')
+        self.add_argument('--apikey', dest='API_KEY', help='the API key to use'
+                         )
         self.add_argument('--schema_dir', default=None,
-                  help='directory to use for API schemas (optional)')
+                          help='directory to use for API schemas (optional)')
 
     def handle(self, args):
         for meta in db.metadata.find():
@@ -66,7 +66,7 @@ def validate_api(abbr, schema_dir=None):
 
         json_response = scrapelib.urlopen(url)
         validictory.validate(json.loads(json_response), bill_schema,
-                                 validator_cls=APIValidator)
+                             validator_cls=APIValidator)
 
     legislator_schema = get_json_schema("legislator", schema_dir)
     for legislator in db.legislators.find(spec):

billy/core/__init__.py

@@ -14,8 +14,8 @@
 
 base_arg_parser = argparse.ArgumentParser(add_help=False)
 
-global_group = base_arg_parser.add_argument_group('global settings',
-                              'settings that apply to all billy commands')
+global_group = base_arg_parser.add_argument_group(
+    'global settings', 'settings that apply to all billy commands')
 
 global_group.add_argument('--mongo_host', help='mongo host', dest='MONGO_HOST')
 global_group.add_argument('--mongo_port', help='mongo port', dest='MONGO_PORT')

billy/ext/ansistrm.py

@@ -64,7 +64,7 @@ def emit(self, record):
             self.handleError(record)
 
     if os.name != 'nt':
-        def output_colorized(self, message):
+        def output_colorized(self, message):    # NOQA
             self.stream.write(message)
     else:
         import re
@@ -81,7 +81,7 @@ def output_colorized(self, message):
             7: 0x07,    # white
         }
 
-        def output_colorized(self, message):
+        def output_colorized(self, message):            # NOQA
             parts = self.ansi_esc.split(message)
             write = self.stream.write
             h = None

billy/importers/bills.py

@@ -230,8 +230,7 @@ def track_version(bill, version):
            'chamber': bill['chamber'],
            'bill_id': bill['bill_id'],
            'subjects': bill.get('subjects', []),
-           'sponsors': [s['leg_id'] for s in bill['sponsors'] if s['leg_id']]
-          }
+           'sponsors': [s['leg_id'] for s in bill['sponsors'] if s['leg_id']]}
     # insert or update this document
     db.tracked_versions.update({'_id': version['doc_id']}, {'$set': doc},
                                upsert=True)
@@ -378,10 +377,10 @@ def _match_legislator(name):
 
         # passed & signed dates
         if (not dates['passed_upper'] and action['actor'] == 'upper'
-            and 'bill:passed' in action['type']):
+                and 'bill:passed' in action['type']):
             dates['passed_upper'] = adate
         elif (not dates['passed_lower'] and action['actor'] == 'lower'
-            and 'bill:passed' in action['type']):
+                and 'bill:passed' in action['type']):
             dates['passed_lower'] = adate
         elif (not dates['signed'] and 'governor:signed' in action['type']):
             dates['signed'] = adate
@@ -397,7 +396,7 @@ def _match_legislator(name):
 
                 delta = abs(vote['date'] - action['date'])
                 if (delta < datetime.timedelta(hours=20) and
-                    vote['chamber'] == action['actor']):
+                        vote['chamber'] == action['actor']):
                     if action_attached:
                         # multiple votes match, we can't guess
                         action.pop('related_votes', None)

billy/importers/committees.py

@@ -27,12 +27,12 @@ def import_committees_from_legislators(current_term, abbr):
 
     # for all current legislators
     for legislator in db.legislators.find({'roles': {'$elemMatch': {
-        'term': current_term, settings.LEVEL_FIELD: abbr}}}):
+            'term': current_term, settings.LEVEL_FIELD: abbr}}}):
 
         # for all committee roles
         for role in legislator['roles']:
             if (role['type'] == 'committee member' and
-                'committee_id' not in role):
+                    'committee_id' not in role):
 
                 spec = {settings.LEVEL_FIELD: abbr,
                         'chamber': role['chamber'],
@@ -47,7 +47,7 @@ def import_committees_from_legislators(current_term, abbr):
                     committee['_type'] = 'committee'
                     # copy LEVEL_FIELD from legislator to committee
                     committee[settings.LEVEL_FIELD] = \
-                            legislator[settings.LEVEL_FIELD]
+                        legislator[settings.LEVEL_FIELD]
                     committee['members'] = []
                     committee['sources'] = []
                     if 'subcommittee' not in committee:
@@ -118,8 +118,8 @@ def import_committee(data, current_session, current_term):
 
         for role in legislator['roles']:
             if (role['type'] == 'committee member' and
-                role['term'] == current_term and
-                role.get('committee_id') == committee['_id']):
+                    role['term'] == current_term and
+                    role.get('committee_id') == committee['_id']):
                 # if the position hadn't been copied over before, copy it now
                 if role.get('position') != member['role']:
                     role['position'] = member['role']
@@ -131,8 +131,7 @@ def import_committee(data, current_session, current_term):
                         'term': current_term,
                         'chamber': committee['chamber'],
                         'committee_id': committee['_id'],
-                        'position': member['role']
-                       }
+                        'position': member['role']}
             # copy over all necessary fields from committee
             new_role[settings.LEVEL_FIELD] = committee[settings.LEVEL_FIELD]
 

billy/importers/legislators.py

@@ -31,8 +31,7 @@ def ensure_indexes():
                                 name='role_and_name_parts')
     db.legislators.ensure_index([(settings.LEVEL_FIELD, pymongo.ASCENDING),
                                  ('active', pymongo.ASCENDING),
-                                 ('chamber', pymongo.ASCENDING),
-                                ])
+                                 ('chamber', pymongo.ASCENDING), ])
 
 
 def import_legislators(abbr, data_dir):
@@ -70,8 +69,9 @@ def activate_legislators(current_term, abbr):
     Sets the 'active' flag on legislators and populates top-level
     district/chamber/party fields for currently serving legislators.
     """
-    for legislator in db.legislators.find({'roles': {'$elemMatch':
-             {settings.LEVEL_FIELD: abbr, 'term': current_term}}}):
+    for legislator in db.legislators.find(
+        {'roles': {'$elemMatch':
+                   {settings.LEVEL_FIELD: abbr, 'term': current_term}}}):
         active_role = legislator['roles'][0]
 
         if not active_role.get('end_date'):
@@ -88,21 +88,16 @@ def deactivate_legislators(current_term, abbr):
 
     # legislators without a current term role or with an end_date
     for leg in db.legislators.find(
-        {'$or': [
-            {'roles': {'$elemMatch':
-                       {'term': {'$ne': current_term},
-                        'type': 'member',
-                          settings.LEVEL_FIELD: abbr,
-                       }},
-            },
-            {'roles': {'$elemMatch':
-                       {'term': current_term,
-                        'type': 'member',
-                          settings.LEVEL_FIELD: abbr,
-                        'end_date': {'$ne':None}}},
-            },
-
-        ]}):
+            {'$or': [
+                {'roles': {'$elemMatch': {
+                    'term': {'$ne': current_term},
+                    'type': 'member', settings.LEVEL_FIELD: abbr}}},
+                {'roles': {'$elemMatch': {
+                    'term': current_term,
+                    'type': 'member',
+                    settings.LEVEL_FIELD: abbr,
+                    'end_date': {'$ne':None}}}}
+            ]}):
 
         if 'old_roles' not in leg:
             leg['old_roles'] = {}