Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
660 lines (564 sloc) 26.5 KB
#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
"""Metamodul som korsrefererar nyckelord som hittats i domslut, andra
beslut, lagkommentarswikitext, osv"""
# system libraries
import logging
import sys, os, re, shutil
from collections import defaultdict
from pprint import pprint
from time import time, sleep
from datetime import datetime
from tempfile import mktemp
import xml.etree.cElementTree as ET
import xml.etree.ElementTree as PET
# 3rdparty libs
from configobj import ConfigObj
from genshi.template import TemplateLoader
from rdflib.Graph import Graph
except ImportError:
from rdflib import Graph
from rdflib import Literal, Namespace, URIRef, RDF, RDFS
# my libs
import Util
import LegalSource
from DispatchMixin import DispatchMixin
from SesameStore import SesameStore
__version__ = (1,6)
__author__ = u"Staffan Malmgren <>"
__shortdesc__ = u"Nyckelord/sökord"
__moduledir__ = "keyword"
log = logging.getLogger(__moduledir__)
if not os.path.sep in __file__:
__scriptdir__ = os.getcwd()
__scriptdir__ = os.path.dirname(__file__)
MW_NS = "{}"
XHT2_NS = "{}"
# module global utility functions
def keyword_to_uri(keyword):
return "" % keyword.replace(" ", "_")
def uri_to_keyword(uri):
return uri.replace("","").replace("_", " ")
re_firstchar = re.compile(r'(\w)', re.UNICODE).search
import mechanize
class HeadRequest(mechanize.Request):
def get_method(self):
return "HEAD"
class KeywordDownloader(LegalSource.Downloader):
def _get_module_dir(self):
return __moduledir__
def _store_select(self,query):
"""Send a SPARQL formatted SELECT query to the Sesame
store. Returns the result as a list of dicts"""
store = SesameStore(self.config['triplestore'], self.config['repository'])
results =
res = []
tree = ET.fromstring(results)
for row in tree.findall(".//{}result"):
d = {}
for element in row:
key = element.attrib['name']
value = element[0].text
d[key] = value
return res
def _store_run_query(self, queryfile, **kwargs):
sq = open(queryfile).read() % kwargs
return self._store_select(sq)
def DownloadAll(self):
# Get all "term sets" (used dct:subject Objects, wiki pages
# describing legal concepts, swedish wikipedia pages...)
terms = defaultdict(dict)
subject_triples_sfs = True
subject_triples = True
wiki_terms = True
wikipedia_terms = True
# 1) Query the RDF DB for all dct:subject triples (is this
# semantically sensible for a "download" action -- the content
# isn't really external?) -- term set "subjects" (these come
# from both court cases and legal definitions in law text)
store = SesameStore(self.config['triplestore'], self.config['repository'])
if subject_triples_sfs:
results = self._store_run_query("sparql/subject_triples_sfs.sq")
for row in results:
subj = row["subj"]
if subj.startswith("http://"):
subj = uri_to_keyword(subj)
subj = subj[0].upper() + subj[1:]
# for sanity: set max length of a subject to 100 chars
subj = subj[:100]
terms[subj][u'subjects'] = True
log.debug("Retrieved subject terms from RDF graph <urn:x-local:sfs>, got %s terms" % len(terms))
if subject_triples:
# for the dv and arn contexts, we should only use subjects
# that appears more than once:
results = self._store_run_query("sparql/subject_triples.sq")
potential_subj = defaultdict(int)
for row in results:
subj = row["subj"]
if subj.startswith("http://"):
# we should really select ?uri rdfs:label ?label instead of munging the URI
subj = uri_to_keyword(subj)
# legacy triples
subj = subj[0].upper() + subj[1:] # uppercase first letter and leave the rest alone
# for sanity: set max length of a subject to 100 chars
subj = subj[:100]
potential_subj[subj] += 1
for (subj,cnt) in potential_subj.items():
if cnt > 1:
terms[subj][u'subjects'] = True
log.debug("Retrieved non-unique subject terms from other RDF graphs, got %s terms" % len(terms))
if wiki_terms:
# print repr(terms.keys()[:10])
# 2) Download the dump from
# -- term set "wiki"
self.browser.set_handle_robots(False) # we can ignore our own robots.txt"")
xml = ET.parse(self.browser.response())
wikinamespaces = []
for ns_el in xml.findall("//"+MW_NS+"namespace"):
for page_el in xml.findall(MW_NS+"page"):
title = page_el.find(MW_NS+"title").text
if title == "Huvudsida":
if ":" in title and title.split(":")[0] in wikinamespaces:
continue # only process pages in the main namespace
if title.startswith("SFS/"):
terms[title][u'wiki'] = True
log.debug("Retrieved subject terms from wiki, now have %s terms" % len(terms))
if wikipedia_terms:
# 3) Download the Wikipedia dump from
# -- term set "wikipedia"
# FIXME: only download when needed
url = ""
do_retrieve = True
localfile = self.download_dir+"/svwiki-latest-all-titles-in-ns0.gz"
if os.path.exists(localfile):
# conditional GET through a ridiculous amount of work
req = HeadRequest(url)
resp =
import locale
tmplocale = locale.getlocale()
last_modified = datetime.strptime(["Last-modified"],
"%a, %d %b %Y %H:%M:%S %Z")
local_modified = datetime.fromtimestamp(os.path.getmtime(localfile))
if local_modified>last_modified:
log.debug("Not fetching %s, local file is newer" % url)
do_retrieve = False
if do_retrieve:
self.browser.retrieve("", self.download_dir+"/svwiki-latest-all-titles-in-ns0.gz")
except Exception:
from gzip import GzipFile
start = time()
wikipediaterms = GzipFile(self.download_dir+"/svwiki-latest-all-titles-in-ns0.gz")
for utf8_term in wikipediaterms:
term = utf8_term.decode('utf-8').strip()
if term in terms:
terms[term][u'wikipedia'] = True
log.debug("Retrieved terms from wikipedia, now have %s terms (%.3f sec)" % (len(terms), time()-start))
# 4) Download all pages from Jureka, probably by starting at
# pageid = 1 and incrementing until done -- term set "jureka"
# Possible future term sets:
# * Rikstermdatabasen
# * various gov websites
# - SKV:
# Store all terms under downloaded/[t]/[term] (for wikipedia,
# store only those terms that occur in any of the other term
# sets). The actual content of each text file contains one
# line for each term set the term occurs in.
for term in terms:
if not term:
firstletter = re_firstchar(term).group(0)
outfile = u"%s/%s/%s.txt" % (self.download_dir, firstletter, term)
if sys.platform != "win32":
outfile = outfile.replace(u'\u2013','--').replace(u'\u2014','---').replace(u'\u2022',u'·').replace(u'\u201d', '"').replace(u'\x96','--').encode("latin-1")
tmpfile = mktemp()
f = open(tmpfile,"w")
for termset in sorted(terms[term]):
except IOError:
log.warning("IOError: Could not write term set file for term '%s'" % term)
except WindowsError:
log.warning("WindowsError: Could not write term set file for term '%s'" % term)
def DownloadNew(self):
# Same as above, except use http if-modified-since to avoid
# downloading swedish wikipedia if not updated. Jureka uses a
# page id parameter, so check if there are any new ones.
class KeywordParser(LegalSource.Parser):
def Parse(self,basefile,infile,config):
# for a base name (term), create a skeleton xht2 file
# containing a element of some kind for each term set this
# term occurs in.
baseuri = keyword_to_uri(basefile)
fp = open(infile,"r")
termsets = fp.readlines()
root = ET.Element("html")
root.set("xml:base", baseuri)
root.set("xmlns", '')
root.set("xmlns:dct", Util.ns['dct'])
head = ET.SubElement(root, "head")
title = ET.SubElement(head, "title")
title.text = basefile
body = ET.SubElement(root,"body")
heading = ET.SubElement(body,"h")
heading.set("property", "dct:title")
heading.text = basefile
if 'wikipedia\n' in termsets:
p = ET.SubElement(body,"p")
p.attrib['class'] = 'wikibox'
p.text = 'Begreppet '
a = ET.SubElement(p,"a")
a.attrib['href'] = '' + basefile.replace(" ","_")
a.text = basefile
a.tail = ' finns beskrivet i '
a = ET.SubElement(p,"a")
a.attrib['href'] = ''
a.text = 'svenska Wikipedia'
return ET.tostring(root, encoding="utf-8")
class KeywordManager(LegalSource.Manager):
__parserClass = KeywordParser
re_tagstrip = re.compile(r'<[^>]*>')
def __init__(self):
# we use the display_title function
import SFS
self.sfsmgr = SFS.SFSManager()
def _get_module_dir(self):
return __moduledir__
def _file_to_basefile(self,f):
return os.path.splitext(f.split(os.sep,3)[3])[0].replace("\\","/")
#return os.path.splitext(os.path.normpath(f).split(os.sep)[-1])[0]
def _build_mini_rdf(self):
termdir = os.path.sep.join([self.baseDir, self.moduleDir, u'parsed'])
minixmlfile = os.path.sep.join([self.baseDir, self.moduleDir, u'parsed', u'rdf-mini.xml'])
ntfile = os.path.sep.join([self.baseDir, self.moduleDir, u'parsed', u'rdf.nt'])
files = list(Util.listDirs(termdir, ".xht2"))
if self._outfile_is_newer(files,minixmlfile):"Not regenerating RDF/XML files")
return"Making a mini graph")
SKOS = Namespace(Util.ns['skos'])
DCT = Namespace(Util.ns['dct'])
mg = Graph()
for key, value in Util.ns.items():
mg.bind(key, Namespace(value));
for f in files:
basefile = os.path.splitext(os.path.normpath(f).split(os.sep)[-1])[0]
termuri = keyword_to_uri(basefile)
mg.add((URIRef(termuri), RDF.type, SKOS['Concept']))
mg.add((URIRef(termuri), DCT['title'], Literal(basefile, lang="sv")))
# Check to see if we have a data/wiki/parsed/[term].xht2 file, and if so, read it's first line
wikifile = Util.relpath(os.path.sep.join([self.baseDir, 'wiki', u'parsed', basefile + '.xht2']))
if os.path.exists(wikifile):
# log.debug("%s exists" % wikifile)
# use the first <p> of the wiki page as a short description
tree = ET.parse(wikifile)
firstpara = tree.find("//"+XHT2_NS+"p")
if firstpara != None: # redirects and empty pages lack <p> tags alltogether. Which works out just fine
xmldesc = ET.tostring(firstpara, encoding='utf-8').decode('utf-8')
textdesc = Util.normalizeSpace(self.re_tagstrip.sub('',xmldesc))
# log.debug(u"%s has desc %s" % (basefile, textdesc))
mg.add((URIRef(termuri), DCT['description'], Literal(textdesc, lang="sv")))"Serializing the minimal graph")
tempfile = mktemp()
f = open(tempfile, 'w')
Util.replace_if_different(tempfile,minixmlfile)"Serializing to NTriples")
tempfile = mktemp()
f = open(tempfile, 'w')
# The nt serializer is broken (
nt_utf8 = mg.serialize(format="nt").decode('utf-8')
for c in nt_utf8:
if ord(c) > 127:
f.write('\u%04X' % ord(c))
def _htmlFileName(self,basefile):
"""Returns the generated, browser-ready XHTML 1.0 file name for the given basefile"""
if not isinstance(basefile, unicode):
raise Exception("WARNING: _htmlFileName called with non-unicode name")
return u'%s/%s/generated/%s.html' % (self.baseDir, self.moduleDir,basefile.replace(" ","_"))
def DownloadAll(self):
d = KeywordDownloader(self.config)
def DownloadNew(self):
d = KeywordDownloader(self.config)
def ParseAll(self):
intermediate_dir = os.path.sep.join([self.baseDir, __moduledir__, u'downloaded'])
self._do_for_all(intermediate_dir, '.txt',self.Parse)
def Parse(self,basefile,verbose=False, wiki_keyword=False):
if verbose:
print "Setting verbosity"
start = time()
infile = os.path.sep.join([self.baseDir, __moduledir__, 'downloaded', basefile]) + ".txt"
if (not os.path.exists(infile)) and wiki_keyword:
fp = open(infile,"w")
basefile = basefile.replace(":","/")
outfile = os.path.sep.join([self.baseDir, __moduledir__, 'parsed', basefile]) + ".xht2"
force = self.config[__moduledir__]['parse_force'] == 'True'
if not force and self._outfile_is_newer([infile],outfile):
p = self.__parserClass()
p.verbose = verbose
keyword = basefile.split("/",1)[1]
parsed = p.Parse(keyword,infile,self.config)
tmpfile = mktemp()
out = file(tmpfile, "w")
os.utime(outfile,None)'%s: OK (%.3f sec)', basefile,time()-start)
def _generateAnnotations(self,annotationfile,basefile):
start = time()
keyword = basefile.split("/",1)[1]
# note: infile is e.g. parsed/K/Konsument.xht2, but outfile is generated/Konsument.html
infile = Util.relpath(self._xmlFileName(basefile))
outfile = Util.relpath(self._htmlFileName(keyword))
# Use SPARQL queries to create a rdf graph (to be used by the
# xslt transform) containing enough information about all
# cases using this term, as well as the wiki authored
# dct:description for this term.
# For proper SPARQL escaping, we need to change å to \u00E5
# etc (there probably is a neater way of doing this).
esckeyword = ''
for c in keyword:
if ord(c) > 127:
esckeyword += '\u%04X' % ord(c)
esckeyword += c
escuri = keyword_to_uri(esckeyword)
sq = """
PREFIX dct:<>
PREFIX rdfs:<>
PREFIX rinfo:<>
SELECT ?desc
WHERE { ?uri dct:description ?desc . ?uri rdfs:label "%s"@sv }
""" % esckeyword
wikidesc = self._store_select(sq)
log.debug(u'%s: Selected %s descriptions (%.3f sec)', basefile, len(wikidesc), time()-start)
sq = """
PREFIX dct:<>
PREFIX rdfs:<>
PREFIX rinfo:<>
GRAPH <urn:x-local:sfs> {
{ ?uri dct:subject <%s> .
?baseuri dct:title ?label .
?uri dct:isPartOf ?x . ?x dct:isPartOf ?baseuri
?uri dct:subject <%s> .
?baseuri dct:title ?label .
?uri dct:isPartOf ?x . ?x dct:isPartOf ?y . ?y dct:isPartOf ?baseuri
?uri dct:subject <%s> .
?baseuri dct:title ?label .
?uri dct:isPartOf ?x . ?x dct:isPartOf ?y . ?x dct:isPartOf ?z . ?z dct:isPartOf ?baseuri
?uri dct:subject <%s> .
?baseuri dct:title ?label .
?uri dct:isPartOf ?x . ?x dct:isPartOf ?y . ?x dct:isPartOf ?z . ?z dct:isPartOf ?w . ?w dct:isPartOf ?baseuri
""" % (escuri,escuri,escuri,escuri)
#print sq
legaldefinitioner = self._store_select(sq)
log.debug(u'%s: Selected %d legal definitions (%.3f sec)', basefile, len(legaldefinitioner), time()-start)
sq = """
PREFIX dct:<>
PREFIX rdfs:<>
PREFIX rinfo:<>
PREFIX rinfoex:<>
SELECT ?uri ?id ?desc
GRAPH <urn:x-local:dv> {
?uri dct:description ?desc .
?uri dct:identifier ?id .
?uri dct:subject <%s>
?uri dct:description ?desc .
?uri dct:identifier ?id .
?uri dct:subject "%s"@sv
GRAPH <urn:x-local:arn> {
?uri dct:description ?desc .
?uri rinfoex:arendenummer ?id .
?uri dct:subject "%s"@sv
""" % (escuri,esckeyword,esckeyword)
# Maybe we should handle <urn:x-local:arn> triples here as well?
rattsfall = self._store_select(sq)
log.debug(u'%s: Selected %d legal cases (%.3f sec)', basefile, len(rattsfall), time()-start)
root_node = PET.Element("rdf:RDF")
for prefix in Util.ns:
if prefix == "xht2":
# This avoids a bug where the xmlns:xht2 is defined
# twice. unfortunately, it results in the xht2 ns not
# being bound to a prefix, so it shows up like
# <ns0:div> in the markup, but that should be ok.
PET._namespace_map[Util.ns[prefix]] = prefix
root_node.set("xmlns:" + prefix, Util.ns[prefix])
main_node = PET.SubElement(root_node, "rdf:Description")
main_node.set("rdf:about", keyword_to_uri(keyword))
for d in wikidesc:
desc_node = PET.SubElement(main_node, "dct:description")
xhtmlstr = "<xht2:div xmlns:xht2='%s'>%s</xht2:div>" % (Util.ns['xht2'], d['desc'])
xhtmlstr = xhtmlstr.replace(' xmlns=""','')
for r in rattsfall:
subject_node = PET.SubElement(main_node, "dct:subject")
rattsfall_node = PET.SubElement(subject_node, "rdf:Description")
id_node = PET.SubElement(rattsfall_node, "dct:identifier")
id_node.text = r['id']
desc_node = PET.SubElement(rattsfall_node, "dct:description")
desc_node.text = r['desc']
for l in legaldefinitioner:
subject_node = PET.SubElement(main_node, "rinfoex:isDefinedBy")
rattsfall_node = PET.SubElement(subject_node, "rdf:Description")
id_node = PET.SubElement(rattsfall_node, "rdfs:label")
#id_node.text = "%s %s" % (l['uri'].split("#")[1], l['label'])
id_node.text = self.sfsmgr.display_title(l['uri'])
tree = PET.ElementTree(root_node)
tmpfile = mktemp()
tree.write(tmpfile, encoding="utf-8")
log.debug("Saving annotation file %s "% annotationfile)
# Update timestamp -- really makes Util.replace_if_different
# redundant Ideally we should change the timestamp to same as
# the newest dependency plus one second?
def Generate(self,basefile):
start = time()
infile = Util.relpath(self._xmlFileName(basefile))
keyword = basefile.split("/",1)[1]
outfile = Util.relpath(self._htmlFileName(keyword))
annotations = "%s/%s/intermediate/%s.ann.xml" % (self.baseDir, self.moduleDir, basefile)
# This was used as a dependency, but it's unneccesary with the new dependency handling
# terms = "%s/%s/parsed/rdf-mini.xml" % (self.baseDir, self.moduleDir)
force = (self.config[__moduledir__]['generate_force'] == 'True')
dependencies = self._load_deps(basefile)
wiki_comments = "data/wiki/parsed/%s.xht2" % basefile.split("/",1)[-1]
if os.path.exists(wiki_comments):
if not force and self._outfile_is_newer(dependencies,annotations):
if os.path.exists(self._depsFileName(basefile)):
log.debug(u"%s: All %s dependencies untouched in rel to %s" %
(basefile, len(dependencies), Util.relpath(annotations)))
log.debug(u"%s: Has no dependencies" % basefile)
else:"%s: Generating annotation file", basefile)
if time()-start > 5:"openrdf-sesame is getting slow, reloading")
cmd = "curl -u %s:%s http://localhost:8080/manager/reload?path=/openrdf-sesame" % (self.config['tomcatuser'], self.config['tomcatpassword'])
sleep(0.5) # let sesame catch it's breath
if not force and self._outfile_is_newer([infile,annotations],outfile):
log.debug(u"%s: Överhoppad", basefile)
# xsltproc silently fails to open files through the document()
# functions if the filename has non-ascii
# characters. Therefore, we copy the annnotation file to a
# separate temp copy first.
tmpfile = mktemp()
# FIXME: create a relative version of annotations, instead of
# hardcoding self.baseDir like below
params = {'annotationfile':tmpfile.replace("\\","/")}
Util.transform(__scriptdir__ + "/xsl/keyword.xsl",
parameters = params,
Util.robust_remove(tmpfile)'%s: OK (%s, %.3f sec)', basefile, outfile, time()-start)
def GenerateAll(self):
parsed_dir = os.path.sep.join([self.baseDir, __moduledir__, u'parsed'])
def RelateAll(self):
# This LegalSource have no triples of it's own
# super(KeywordManager,self).RelateAll()
# not yet ready for prime time
# def _build_indexpages(self, by_pred_obj, by_subj_pred):
# documents = defaultdict(lambda:defaultdict(list))
# pagetitles = {}
# pagelabels = {}
# type_pred = Util.ns['rdf']+'type'
# type_obj = Util.ns['skos']+'Concept'
# title_pred = Util.ns['dct']+'title'
# for subj in by_pred_obj[type_pred][type_obj]:
# title = by_subj_pred[subj][title_pred]
# letter = title[0].lower()
# pagetitles[letter] = u'Begrepp som börjar på "%s"' % letter.upper()
# pagelabels[letter] = letter.upper()
# documents[u'Inledningsbokstav'][letter].append({'uri':subj,
# 'sortkey':title.lower(),
# 'title':title})
# for category in documents.keys():
# for pageid in documents[category].keys():
# outfile = "%s/%s/generated/index/%s.html" % (self.baseDir, self.moduleDir, pageid)
# title = pagetitles[pageid]
# self._render_indexpage(outfile,title,documents,pagelabels,category,pageid)
# if pageid == 'a': # make index.html
# outfile = "%s/%s/generated/index/index.html" % (self.baseDir, self.moduleDir)
# self._render_indexpage(outfile,title,documents,pagelabels,category,pageid)
if __name__ == "__main__":
import logging.config
KeywordManager.__bases__ += (DispatchMixin,)
mgr = KeywordManager()