Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
581 lines (495 sloc) 22.5 KB
#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
"""Modul för att översätta innehållet i en Mediawikiinstans till RDF-triples"""
# system
import sys, os, re
import xml.etree.cElementTree as ET
import xml.etree.ElementTree as PET
import logging
from time import time
from tempfile import mktemp
from urllib import quote
from urllib2 import HTTPError
import random
from pprint import pprint
import traceback
# 3rdparty
from configobj import ConfigObj
from rdflib import Namespace
import wikimarkup
# mine
import Util
import LegalSource
from DispatchMixin import DispatchMixin
from LegalRef import LegalRef, Link
from SFS import FilenameToSFSnr
from SesameStore import SesameStore
import FilebasedTester
log = logging.getLogger(u'ls')
__version__ = (1,6)
__author__ = u"Staffan Malmgren <staffan@tomtebo.org>"
__shortdesc__ = u"Wikikommentarer"
__moduledir__ = "wiki"
log = logging.getLogger(__moduledir__)
# MW_NS = "{http://www.mediawiki.org/xml/export-0.3/}"
MW_NS = "{http://www.mediawiki.org/xml/export-0.10/}"
class WikiDownloader(LegalSource.Downloader):
def __init__(self,config):
super(WikiDownloader,self).__init__(config) # sets config, logging, initializes browser
self.browser.set_handle_robots(False) # we can ignore our own robots.txt
def _get_module_dir(self):
return __moduledir__
def DownloadAll(self):
wikinamespaces = []
# this file is regenerated hourly
url = "https://lagen.nu/wiki-pages-articles.xml"
self.browser.open(url)
xml = ET.parse(self.browser.response())
for ns_el in xml.findall(".//"+MW_NS+"namespace"):
wikinamespaces.append(ns_el.text)
# Get list of currently downloaded pages - if any of those
# does not appear in the XML dump, remove them afterwards
downloaded_files = list(Util.listDirs(Util.relpath(unicode(self.download_dir)),".xml"))
for page_el in xml.findall(MW_NS+"page"):
title = page_el.find(MW_NS+"title").text
if title == "Huvudsida":
continue
if ":" in title and title.split(":")[0] in wikinamespaces:
(namespace, localtitle) = title.split(":",1)
if namespace != "Kategori":
continue # only process pages in the main + Kategori namespace
outfile = "%s/%s.xml" % (self.download_dir, title.replace(":","/"))
outfile = Util.relpath(outfile)
tmpfile = mktemp()
f = open(tmpfile,"w")
f.write(ET.tostring(page_el,encoding="utf-8"))
f.close()
if Util.replace_if_different(tmpfile,outfile):
#log.debug("Dumping %s" % outfile)
pass
else:
#log.debug("Not replacing %s" % outfile)
pass
try:
del downloaded_files[downloaded_files.index(outfile)]
except ValueError:
# Will happen for new files
pass
for f in downloaded_files:
log.debug("Removing %s" % f)
try:
Util.robust_remove(f)
except OSError, e:
log.warning(u"Can't remove %r: %r" % (f, e))
def DownloadNew(self):
self.DownloadAll()
def _downloadSingle(self,term):
# download a single term, for speed
if isinstance(term,unicode):
encodedterm = quote(term.encode('utf-8'))
else:
encodedterm = quote(term)
url = "https://lagen.nu/wiki/Special:Exportera/%s" % encodedterm
# FIXME: if outfile already exist, only save if wiki resource is modified since
outfile = "%s/%s.xml" % (self.download_dir, term.replace(":","/"))
Util.ensureDir(outfile)
log.info("Downloading wiki text for term %s to %s ", term, outfile)
self.browser.retrieve(url,outfile)
class LinkedWikimarkup(wikimarkup.Parser):
def __init__(self, show_toc=True):
super(wikimarkup.Parser, self).__init__()
self.show_toc = show_toc
def parse(self,text):
#print "Running subclassed parser!"
utf8 = isinstance(text, str)
text = wikimarkup.to_unicode(text)
if text[-1:] != u'\n':
text = text + u'\n'
taggedNewline = True
else:
taggedNewline = False
text = self.strip(text)
text = self.removeHtmlTags(text)
text = self.doTableStuff(text)
text = self.parseHorizontalRule(text)
text = self.checkTOC(text)
text = self.parseHeaders(text)
text = self.parseAllQuotes(text)
text = self.replaceExternalLinks(text)
if not self.show_toc and text.find(u"<!--MWTOC-->") == -1:
self.show_toc = False
text = self.formatHeadings(text, True)
text = self.unstrip(text)
text = self.fixtags(text)
text = self.replaceRedirect(text)
text = self.doBlockLevels(text, True)
text = self.unstripNoWiki(text)
text = self.replaceImageLinks(text)
text = self.replaceCategories(text)
text = self.replaceAuthorLinks(text)
text = self.replaceWikiLinks(text)
text = self.removeTemplates(text)
text = text.split(u'\n')
text = u'\n'.join(text)
if taggedNewline and text[-1:] == u'\n':
text = text[:-1]
if utf8:
return text.encode("utf-8")
return text
re_labeled_wiki_link = re.compile(r'\[\[([^\]]*?)\|(.*?)\]\](\w*)') # is the trailing group really needed?
re_wiki_link = re.compile(r'\[\[([^\]]*?)\]\](\w*)')
re_img_uri = re.compile('(https?://[\S]+\.(png|jpg|gif))')
re_template = re.compile(r'{{[^}]*}}')
re_category_wiki_link = re.compile(r'\[\[Kategori:([^\]]*?)\]\]')
re_inline_category_wiki_link = re.compile(r'\[\[:Kategori:([^\]]*?)\|(.*?)\]\]')
re_image_wiki_link = re.compile(r'\[\[Fil:([^\]]*?)\s*\]\]')
re_author_wiki_link = re.compile(ur'\[\[(Användare:[^\]]+?)\|(.*?)\]\]')
def capitalizedLink(self,m):
if m.group(1).startswith('SFS/'):
uri = 'http://rinfo.lagrummet.se/publ/%s' % m.group(1).lower()
else:
uri = 'http://lagen.nu/concept/%s' % Util.ucfirst(m.group(1)).replace(' ','_')
if len(m.groups()) == 3:
# lwl = "Labeled WikiLink"
return '<a class="lwl" href="%s">%s%s</a>' % (uri, m.group(2), m.group(3))
else:
return '<a class="wl" href="%s">%s%s</a>' % (uri, m.group(1), m.group(2))
def categoryLink(self,m):
uri = 'http://lagen.nu/concept/%s' % Util.ucfirst(m.group(1)).replace(' ','_')
if len(m.groups()) == 2:
# lcwl = "Labeled Category WikiLink"
return '<a class="lcwl" href="%s">%s</a>' % (uri, m.group(2))
else:
# cwl = "Category wikilink"
return '<a class="cwl" href="%s">%s</a>' % (uri, m.group(1))
def hiddenLink(self,m):
uri = 'http://lagen.nu/concept/%s' % Util.ucfirst(m.group(1)).replace(' ','_')
return '<a class="hcwl" rel="dct:subject" href="%s"/>' % uri
def imageLink(self,m):
uri = 'https://lagen.nu/w/images/%s' % m.group(1).strip()
return '<img class="iwl" src="%s" />' % uri
def authorLink(self,m):
uri = u'https://lagen.nu/wiki/%s' % Util.ucfirst(m.group(1)).replace(' ','_')
return '<a class="awl" href="%s">%s</a>' % (uri,m.group(2))
def replaceWikiLinks(self,text):
# print "replacing wiki links: %s" % text[:30]
text = self.re_labeled_wiki_link.sub(self.capitalizedLink, text)
text = self.re_wiki_link.sub(self.capitalizedLink, text)
return text
def replaceImageLinks(self,text):
# emulates the parser when using$ wgAllowExternalImages
text = self.re_img_uri.sub('<img src="\\1"/>',text)
# handle links like [[Fil:SOU_2003_99_s117.png]]
text = self.re_image_wiki_link.sub(self.imageLink, text)
return text
def replaceAuthorLinks(self,text):
# links to author descriptions should point directly to the wiki
return self.re_author_wiki_link.sub(self.authorLink, text)
def removeTemplates(self,text):
# removes all usage of templates ("{{DISPLAYTITLE:Avtalslagen}}" etc)
return self.re_template.sub('',text)
def replaceCategories(self,text):
# inline links ("Inom [[:Kategori:Allmän avtalsrätt|Allmän avtalsrätt]] studerar man...")
text = self.re_inline_category_wiki_link.sub(self.categoryLink,text)
# Normal category links - replace these with hidden RDFa typed links
text = self.re_category_wiki_link.sub(self.hiddenLink,text)
return text
re_redirect = re.compile("^#REDIRECT ")
def replaceRedirect(self, text):
return self.re_redirect.sub("Se ", text)
class WikiParser(LegalSource.Parser):
re_anchors = re.compile('(<a.*?</a>)',re.DOTALL)
re_anchor = re.compile('<a[^>]*>(.*)</a>',re.DOTALL)
re_tags = re.compile('(</?[^>]*>)',re.DOTALL)
re_sfs_uri = re.compile('https?://[^/]*lagen.nu/(\d+):(.*)')
re_dom_uri = re.compile('https?://[^/]*lagen.nu/dom/(.*)')
# This is getting complex... we should write some test cases.
def Parse(self,basefile,infile,config=None):
xml = ET.parse(open(infile))
wikitext = xml.find(".//"+MW_NS+"text").text
#if wikitext:
# return wikitext.encode('iso-8859-1',"replace")
#else:
# return ''
return self.parse_wikitext(basefile,wikitext)
def parse_wikitext(self,basefile,wikitext):
p = LinkedWikimarkup(show_toc=False)
html = p.parse(wikitext)
# the output from wikimarkup is less than ideal...
html = html.replace("&", "&amp;");
html = '<div>'+html+'</div>';
try:
xhtml = ET.fromstring(html.encode('utf-8'))
except SyntaxError:
log.warn("%s: wikiparser did not return well-formed markup (working around)" % basefile)
# print u"Invalid markup:\n%s" % html
tmpfilename = mktemp()
fp = open(tmpfilename,"w")
fp.write(html.encode('utf-8'))
fp.close()
log.debug("Saved invalid HTML as %s" % tmpfilename)
tidied = Util.tidy(html.encode('utf-8')).replace(' xmlns="http://www.w3.org/1999/xhtml"','').replace('&nbsp;','&#160;')
# print "Valid markup:\n%s" % tidied
xhtml = ET.fromstring(tidied.encode('utf-8')).find("body/div")
# p = LegalRef(LegalRef.LAGRUM)
Util.indent_et(xhtml)
#print ET.tostring(xhtml,'utf-8').decode('utf-8')
p = LegalRef(LegalRef.LAGRUM, LegalRef.KORTLAGRUM, LegalRef.FORARBETEN, LegalRef.RATTSFALL)
# find out the URI that this wikitext describes
if basefile.startswith("SFS/"):
filename = basefile.split("/",1)[1]
if len(filename.split("/")) > 2:
filename = filename.rsplit("/",1)[0]
sfs = FilenameToSFSnr(filename)
nodes = p.parse(sfs)
uri = nodes[0].uri
rdftype = None
else:
# concept == "begrepp"
uri = "http://lagen.nu/concept/" + basefile.replace(" ","_")
rdftype = "skos:Concept"
log.debug(" URI: %s" % uri)
root = ET.Element("html")
root.set("xmlns", 'http://www.w3.org/2002/06/xhtml2/')
root.set("xmlns:dct", Util.ns['dct'])
root.set("xmlns:rdf", Util.ns['rdf'])
root.set("xmlns:rdfs", Util.ns['rdfs'])
root.set("xmlns:skos", Util.ns['skos'])
root.set("xml:lang", "sv")
head = ET.SubElement(root,"head")
title = ET.SubElement(head,"title")
title.text = basefile
body = ET.SubElement(root,"body")
body.set("about", uri)
if rdftype:
body.set("typeof", "skos:Concept")
heading = ET.SubElement(body, "h")
heading.set("property", "rdfs:label")
heading.text = basefile
main = ET.SubElement(body, "div")
main.set("property", "dct:description")
main.set("datatype", "rdf:XMLLiteral")
current = main
currenturi = uri
for child in xhtml:
if not rdftype and child.tag in ('h1','h2','h3','h4','h5','h6'):
nodes = p.parse(child.text,uri)
try:
suburi = nodes[0].uri
currenturi = suburi
log.debug(" Sub-URI: %s" % suburi)
h = ET.SubElement(body, child.tag)
h.text = child.text
current = ET.SubElement(body,"div")
current.set("about", suburi)
current.set("property", "dct:description")
current.set("datatype", "rdf:XMLLiteral")
except AttributeError:
log.warning(u'%s är uppmärkt som en rubrik, men verkar inte vara en lagrumshänvisning' % child.text)
else:
serialized = ET.tostring(child,'utf-8').decode('utf-8')
separator = ""
while separator in serialized:
separator = "".join(random.sample("ABCDEFGHIJKLMNOPQRSTUVXYZ",6))
markers = {}
res = ""
# replace all whole <a> elements with markers, then
# replace all other tags with markers
for (regex,start) in ((self.re_anchors,'<a'),
(self.re_tags, '<')):
for match in re.split(regex, serialized):
if match.startswith(start):
marker = "{%s-%d}" % (separator,len(markers))
markers[marker] = match
res += marker
else:
res += match
serialized = res
res = ""
# Use LegalRef to parse references, then rebuild a
# unicode string.
parts = p.parse(serialized,currenturi)
for part in parts:
if isinstance(part, Link):
res += u'<a class="lr" href="%s">%s</a>' % (part.uri, part)
else: # just a text fragment
res += part
# restore the replaced markers
for marker, replacement in markers.items():
#print "%s: '%s'" % (marker,Util.normalizeSpace(replacement))
# normalize URIs, and remove 'empty' links
if 'href="https://lagen.nu/"' in replacement:
replacement = self.re_anchor.sub('\\1', replacement)
elif self.re_sfs_uri.search(replacement):
replacement = self.re_sfs_uri.sub('http://rinfo.lagrummet.se/publ/sfs/\\1:\\2',replacement)
elif self.re_dom_uri.search(replacement):
replacement = self.re_dom_uri.sub('http://rinfo.lagrummet.se/publ/rattsfall/\\1',replacement)
#print "%s: '%s'" % (marker,Util.normalizeSpace(replacement))
res = res.replace(marker,replacement)
current.append(ET.fromstring(res.encode('utf-8')))
Util.indent_et(root)
# print ET.tostring(root,'utf-8').decode('utf-8')
res = ET.tostring(root,encoding='utf-8')
return res
class WikiManager(LegalSource.Manager,FilebasedTester.FilebasedTester):
__parserClass = WikiParser
def _get_module_dir(self):
return __moduledir__
def __init__(self):
super(WikiManager,self).__init__()
self.moduleDir = "wiki"
def Download(self,term):
d = WikiDownloader(self.config)
d._downloadSingle(term)
def DownloadAll(self):
d = WikiDownloader(self.config)
d.DownloadAll()
def DownloadNew(self):
d = WikiDownloader(self.config)
d.DownloadNew()
def ParseAll(self):
intermediate_dir = os.path.sep.join([self.baseDir, __moduledir__, u'downloaded'])
self._do_for_all(intermediate_dir, '.xml',self.Parse)
def _file_to_basefile(self,f):
seg = os.path.splitext(f)[0].split(os.sep)
return "/".join(seg[seg.index(__moduledir__)+2:])
def Parse(self, basefile, verbose=False):
if verbose:
print "Setting verbosity"
log.setLevel(logging.DEBUG)
start = time()
basefile = basefile.replace(":","/").replace("\\","/")
categoryfile = os.path.sep.join([self.baseDir, __moduledir__, 'downloaded', 'Kategori', basefile]) + ".xml"
if os.path.exists(categoryfile):
log.debug(u"%s: Skippad", basefile)
return
infile = os.path.sep.join([self.baseDir, __moduledir__, 'downloaded', basefile]) + ".xml"
if "Kategori/" in basefile:
basefile = basefile.replace("Kategori/","")
outfile = os.path.sep.join([self.baseDir, __moduledir__, 'parsed', basefile]) + ".xht2"
force = self.config[__moduledir__]['parse_force'] == 'True'
if not force and self._outfile_is_newer([infile],outfile):
log.debug(u"%s: Överhoppad", basefile)
return
p = self.__parserClass()
p.verbose = verbose
parsed = p.Parse(basefile,infile,self.config)
Util.ensureDir(outfile)
tmpfile = mktemp()
out = file(tmpfile, "w")
out.write(parsed)
out.close()
# Util.indentXmlFile(tmpfile)
Util.replace_if_different(tmpfile,outfile)
os.utime(outfile,None)
(chars,words) = self.wc(parsed)
log.info(u'%s: OK (%.3f sec, %d words, %d chars)', basefile,time()-start, words, chars)
re_tags = re.compile(r'<.*?>')
def wc(self,txt):
txt = self.re_tags.sub('',txt)
return (len(txt), len (txt.split()))
def Generate(self,basefile):
# No pages to generate for this src (pages for
# keywords/concepts are done by Keyword.py)
pass
def GenerateAll(self):
pass
def Relate(self, basefile):
basefile = basefile.replace(":","/")
if basefile.startswith("Kategori/"):
basefile = basefile.replace("Kategori/","")
# each wiki page gets a unique context - this is so that we
# can easily delete its statements once we re-parse and
# re-relate it's content
context = "<urn:x-local:%s:%s>" % (self.moduleDir, quote(basefile.encode('utf-8').replace(" ","_")))
store = SesameStore(self.config['triplestore'], self.config['repository'],context)
infile = os.path.sep.join([self.baseDir, __moduledir__, 'parsed', basefile]) + ".xht2"
# print "loading triples from %r" % Util.relpath(infile)
graph = self._extract_rdfa(Util.relpath(infile))
store.clear()
triples = 0
triples += len(graph)
for key, value in Util.ns.items():
store.bind(key, Namespace(value));
store.add_graph(graph)
store.commit()
log.debug("Related %s: %d triples" % (basefile, triples))
def RelateAll(self,file=None):
# we override LegalSource.RelateAll since we want a different
# context for each wiki page
"""Sammanställer alla wiki-baserade beskrivningstriples och
laddar in dem i systemets triplestore"""
files = list(Util.listDirs(os.path.sep.join([self.baseDir, self.moduleDir, u'parsed']), '.xht2'))
rdffile = os.path.sep.join([self.baseDir, self.moduleDir, u'parsed', u'rdf.nt'])
if self._outfile_is_newer(files,rdffile):
log.info("%s is newer than all .xht2 files, no need to extract" % rdffile)
# FIXME: regardless of this, we should fast-load the store
# with the previously-extracted triples, but this is
# difficult since we use multiple contexts (also see
# below)
return
c = 0
for f in files:
basefile = self._file_to_basefile(f)
try:
self.Relate(basefile)
c += 1
if c % 100 == 0:
log.info("Related %d wiki entries" % c)
except KeyboardInterrupt:
raise
except:
# Handle traceback-loggning ourselves since the
# logging module can't handle source code containing
# swedish characters (iso-8859-1 encoded).
formatted_tb = [x.decode('iso-8859-1') for x in traceback.format_tb(sys.exc_info()[2])]
exception = sys.exc_info()[1]
msg = exception
if isinstance(exception, HTTPError):
msg = repr(exception) + ": " + exception.read()
if not msg:
if isinstance(exception,OSError):
msg = "[Errno %s] %s: %s" % (exception.errno, exception.strerror, exception.filename)
else:
msg = "(Message got lost)"
log.error(u'%r: %s:\nMyTraceback (most recent call last):\n%s%s [%s]' %
(basefile,
sys.exc_info()[0].__name__,
u''.join(formatted_tb),
sys.exc_info()[0].__name__,
msg))
# should we serialize everything to a big .nt file like the
# other LegalSources does? It's a bit more difficult since we
# have different contexts. For now, just touch() the file so
# that the _otfile_is_newer trick works
f = open(rdffile,"w")
f.close()
################################################################
# IMPLEMENTATION OF FilebasedTester interface
################################################################
testparams = {'Parse': {'dir': u'test/Wiki',
'testext':'.txt',
'testencoding':'latin-1',
'answerext':'.xht2',
'answerencoding':'utf-8'},
}
def TestParse(self,data,verbose=None,quiet=None):
# FIXME: Set this from FilebasedTester
if verbose == None:
verbose=False
if quiet == None:
#quiet=True
pass
p = WikiParser()
p.verbose = verbose
res = p.parse_wikitext("Test",data)
if isinstance(res,unicode):
return res
else:
return res.decode('utf-8')
if __name__ == "__main__":
import logging.config
logging.config.fileConfig('etc/log.conf')
WikiManager.__bases__ += (DispatchMixin,)
mgr = WikiManager()
mgr.Dispatch(sys.argv)