Skip to content

Commit

Permalink
new bill loader option to just load recently changed files
Browse files Browse the repository at this point in the history
  • Loading branch information
JoshData committed May 6, 2020
1 parent 761be93 commit a7e09cd
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion parser/bill_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ def main(options):
else:
files = glob.glob(settings.CONGRESS_DATA_PATH + '/*/bills/*/*/data.xml')

if options.filter:
if options.filter and options.filter != "recent":
files = [f for f in files if re.match(options.filter, f)]

log.info('Processing bills: %d files' % len(files))
Expand Down Expand Up @@ -383,6 +383,10 @@ def main(options):

if options.slow:
time.sleep(1)

if options.filter == "recent":
if datetime.fromtimestamp(os.path.getmtime(fname)) < datetime.now() - timedelta(days=1):
continue

tree = etree.parse(fname)
for node in tree.xpath('/bill'):
Expand Down

0 comments on commit a7e09cd

Please sign in to comment.