Find file
Fetching contributors…
Cannot retrieve contributors at this time
116 lines (95 sloc) 4.32 KB
# -*- coding: utf-8 -*-
import string, os, sys
from splithtml import *
import codecs
import re2 as re
import re
regfile = os.path.abspath(os.path.dirname(__file__))+'/usc-regex.txt'
divider="<!-- documentid:"
_DEBUG = False
# To parse the html in data/, documentid:26_12 = 26usc12
def makergxlist(regfile):
rfile =, 'r', 'utf-8')
findreplace = []
for eachline in rfile.readlines():
if (not eachline.isspace()) & (eachline[0] <> '#'):
pattern_replace = unicode(eachline).strip().split('#')
findreplace = [(re.compile(pattern, re.U|re.M), replacement) for pattern, replacement in findreplace]
return findreplace
def subfile(inputfile, findreplace):
outtext = inputfile
if inputfile is None:
return outtext
for counter, pattern_replace in enumerate(findreplace):
outtext, subs = re.subn(pattern_replace[0], pattern_replace[1], outtext)
if _DEBUG and subs > 0:
print pattern_replace[0].pattern
print pattern_replace[1]
print "=" * 30
except KeyboardInterrupt:
print counter, ' substituted: ', subs
print pattern_replace[0], "||||", pattern_replace[1], outtext
return outtext
# Dynamically replace references with links
def parsesections(pattern, pattern_replace, section):
sectionsref =, section)
while sectionsref:
i1 = sectionsref.start(1)
i2 = sectionsref.end(2)
#print "found multiple secs at", i1, "-", i2
section = section[:i1]+re.sub(pattern_replace[0], pattern_replace[1] %, section[i1:i2]) + section[1+i2:]
sectionsref =, section)
return section
def parsenamedacts(pattern, intext):
namedacts = re.findall(pattern, intext)
namedacts = list(set(namedacts))
outtext = intext
for namedact in namedacts:
#outtext = outtext.replace(namedact+r'@/', encode_act(namedact)+r'@/')
outtext = outtext.replace(r'ref-namedact-'+namedact,r'ref-namedact-'+encode_act(namedact))
return outtext
def encode_act(namedact):
#encoding = namedact.translate(None, string.punctuation) # removed because it is fails with unicode objects
encoding = ''.join([char if char not in string.punctuation else '' for char in namedact])
encodelen = str(len(encoding))
encoding = [char if char not in string.lowercase+" " else '' for char in encoding] #[encoding.translate(None, string.lowercase + r' ')]
encoding.append('-' + encodelen)
out = "".join(encoding)
return out
# Loop through sections with various parsing passes
def parsers(uscfiles, findreplace):
parsedfiles = []
for counter, section in enumerate(uscfiles):
#print ""
#print "File", counter
parsedfile = subfile(section, findreplace)
#print parsedfile
# Replace Multiple Section references with links
# Include [^<] to make sure no group is transformed twice
pattern = r'@@@\s[Ss]ections?\s([^<]*?)@@@@@(.*?)@@'
pattern_replace = [r'%s' % u'(\d+\w*(?:\(\w+\))*[-|–]?\d*)([, @])', r'<a href="/laws/target/%s/\1" class="sec">\1</a>\2']
parsedfile = parsesections(pattern, pattern_replace, parsedfile)
#parsedfile = re.sub(r'@of-ref@', r'ref-Title-'+title, parsedfile)
parsedfile = re.sub(r'@@ref-.*?@', r'', parsedfile)
#parsedfile = re.sub(r'ref-title-this', r'ref-title-'+title, parsedfile)
# Encode Named Acts by removing lowercase and non-word characters, and appending the length of the name w/o non-word characters
#pattern = r'@@ref-namedact-(.*?)@@'
pattern = r'/ref-namedact-(.*?)/'
parsedfile = parsenamedacts(pattern, parsedfile)
parsedfile = re.sub(r'@of-ref@', r'ref-title-this', parsedfile)
parsedfile = re.sub(r'@@ref-.*?@', r'', parsedfile)
# Remove remaining @
parsedfile = parsedfile.replace('@','')#.translate(None, '@')
return parsedfiles
def parse(data=False):
uscfiles = [data, ]#.split(divider)#splithtml()
findreplace = makergxlist(regfile)
parsedfiles = parsers(uscfiles, findreplace)
return parsedfiles