From df17f103036a655c0e9dcdde35b5f6d79bb4b58d Mon Sep 17 00:00:00 2001 From: Staffan Malmgren Date: Sat, 5 Jul 2014 22:19:08 +0200 Subject: [PATCH] maybe now all functional and integration tests work? --- doc/examples/firststeps.sh | 42 +++++++++++++++++++++++------------ doc/examples/rfcs.py | 4 ++-- ferenda/documentrepository.py | 22 +++++++++--------- ferenda/elements/elements.py | 11 ++++++--- ferenda/sources/tech/rfc.py | 14 +++++++----- test/functionalDocExamples.py | 12 +++++----- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/doc/examples/firststeps.sh b/doc/examples/firststeps.sh index a732c76b..cf57c8e6 100644 --- a/doc/examples/firststeps.sh +++ b/doc/examples/firststeps.sh @@ -75,7 +75,7 @@ $ ./ferenda-build.py w3c download rdfa-core --loglevel=CRITICAL # begin parse $ ./ferenda-build.py w3c parse rdfa-core -13:04:33 w3c INFO rdfa-core: OK (2.033 sec) +13:04:33 w3c INFO rdfa-core: parse OK (2.033 sec) 13:04:33 root INFO w3c parse finished in 2.053 sec # end parse @@ -100,7 +100,7 @@ $ ./ferenda-build.py w3c parse rdfa-core # begin parse-force $ ./ferenda-build.py w3c parse rdfa-core --force -13:04:38 w3c INFO rdfa-core: OK (2.024 sec) +13:04:38 w3c INFO rdfa-core: parse OK (2.024 sec) 13:04:38 root INFO w3c parse finished in 2.043 sec # end parse-force @@ -111,22 +111,26 @@ $ ./ferenda-build.py w3c parse --all --loglevel=DEBUG 13:04:39 w3c DEBUG ttml1: Starting 13:04:43 w3c DEBUG ttml1: Created data/w3c/parsed/ttml1.xhtml 13:04:45 w3c DEBUG ttml1: 12 triples extracted to data/w3c/distilled/ttml1.rdf -13:04:45 w3c INFO ttml1: OK (5.816 sec) +13:04:45 w3c INFO ttml1: parse OK (5.816 sec) 13:04:45 w3c DEBUG touch-events: Starting 13:04:45 w3c DEBUG touch-events: Created data/w3c/parsed/touch-events.xhtml 13:04:45 w3c DEBUG touch-events: 8 triples extracted to data/w3c/distilled/touch-events.rdf -13:04:45 w3c INFO touch-events: OK (0.486 sec) +13:04:45 w3c INFO touch-events: parse OK (0.486 sec) 13:04:45 w3c DEBUG geolocation-API: Starting 13:04:46 w3c DEBUG geolocation-API: Created data/w3c/parsed/geolocation-API.xhtml 13:04:46 w3c DEBUG geolocation-API: 5 triples extracted to data/w3c/distilled/geolocation-API.rdf -13:04:46 w3c INFO geolocation-API: OK (0.323 sec) +13:04:46 w3c INFO geolocation-API: parse OK (0.323 sec) 13:04:46 root INFO w3c parse finished in 6.662 sec # end parse-all # begin relate-all $ ./ferenda-build.py w3c relate --all 13:04:47 w3c INFO Clearing context http://localhost:8000/dataset/w3c at repository ferenda -13:04:54 w3c INFO Dumped 34 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt +13:04:47 w3c INFO xml-entity-names: relate OK (0.416 sec) +13:04:47 w3c INFO rdfa-core: relate OK (1.429 sec) +13:04:47 w3c INFO emotionml: relate OK (1.897 sec) +13:04:47 w3c INFO MathML3: relate OK (2.494 sec) +13:04:54 w3c INFO Dumped 34 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt (0.123 sec) 13:04:54 root INFO w3c relate finished in 7.655 sec # end relate-all @@ -134,6 +138,10 @@ $ ./ferenda-build.py w3c relate --all $ ./ferenda-build.py w3c makeresources $ find data/rsrc -print data/rsrc +data/rsrc/api +data/rsrc/api/common.json +data/rsrc/api/context.json +data/rsrc/api/terms.json data/rsrc/css data/rsrc/css/ferenda.css data/rsrc/css/main.css @@ -148,10 +156,10 @@ data/rsrc/resources.xml # begin generate-all $ ./ferenda-build.py w3c generate --all -13:04:58 w3c INFO ttml1: OK (2.102 sec) -13:04:59 w3c INFO touch-events: OK (0.112 sec) -13:04:59 w3c INFO rdfa-core: OK (0.220 sec) -13:04:59 w3c INFO geolocation-API: OK (0.100 sec) +13:04:58 w3c INFO ttml1: generate OK (2.102 sec) +13:04:59 w3c INFO touch-events: generate OK (0.112 sec) +13:04:59 w3c INFO rdfa-core: generate OK (0.220 sec) +13:04:59 w3c INFO geolocation-API: generate OK (0.100 sec) 13:04:59 root INFO w3c generate finished in 2.547 sec # end generate-all @@ -160,6 +168,9 @@ $ ./ferenda-build.py w3c toc 13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html 13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html 13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html 13:05:02 w3c INFO Created data/w3c/toc/title/a.html 13:05:02 w3c INFO Created data/w3c/toc/title/c.html 13:05:02 w3c INFO Created data/w3c/toc/title/r.html @@ -185,14 +196,17 @@ $ ./ferenda-build.py w3c all 13:05:07 root INFO w3cstandards.W3CStandards parse finished in 0.010 sec 13:05:07 root INFO w3cstandards.W3CStandards relate: Nothing to do! 13:05:07 root INFO w3cstandards.W3CStandards relate finished in 0.005 sec -13:05:07 w3c INFO ttml1: OK (0.000 sec) -13:05:07 w3c INFO touch-events: OK (0.000 sec) -13:05:07 w3c INFO rdfa-core: OK (0.000 sec) -13:05:07 w3c INFO geolocation-API: OK (0.000 sec) +13:05:07 w3c INFO ttml1: generate OK (0.000 sec) +13:05:07 w3c INFO touch-events: generate OK (0.000 sec) +13:05:07 w3c INFO rdfa-core: generate OK (0.000 sec) +13:05:07 w3c INFO geolocation-API: generate OK (0.000 sec) 13:05:07 root INFO w3cstandards.W3CStandards generate finished in 0.006 sec 13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html 13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html 13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html +13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html 13:05:02 w3c INFO Created data/w3c/toc/title/a.html 13:05:02 w3c INFO Created data/w3c/toc/title/c.html 13:05:02 w3c INFO Created data/w3c/toc/title/r.html diff --git a/doc/examples/rfcs.py b/doc/examples/rfcs.py index 56214a0a..ce812baa 100644 --- a/doc/examples/rfcs.py +++ b/doc/examples/rfcs.py @@ -296,8 +296,8 @@ def facets(self): # begin toc_item def toc_item(self, binding, row): from ferenda.elements import Link - return [row['identifier'] + ": ", - Link(row['title'], + return [row['dcterms_identifier'] + ": ", + Link(row['dcterms_title'], uri=row['uri'])] # end toc_item diff --git a/ferenda/documentrepository.py b/ferenda/documentrepository.py index a86b5ad2..49824771 100644 --- a/ferenda/documentrepository.py +++ b/ferenda/documentrepository.py @@ -1403,7 +1403,7 @@ def relate_all_teardown(cls, config): # relate_all_setup), do the actual bulk upload. if os.path.exists(temp): with util.logtime(log.info, - "Loaded %(triplecount)s triples to context %(context)s from %(tempfile)s in %(elapsed).3f s", + "Loaded %(triplecount)s triples to context %(context)s from %(tempfile)s (%(elapsed).3f sec)", values): store.add_serialized_file(temp, format="nt", context=context) # just to report the number of dumped triples -- may be unneccesary @@ -1414,7 +1414,7 @@ def relate_all_teardown(cls, config): # same contents as the temp file, but this comes directly from # the triplestore with util.logtime(log.info, - "Dumped %(triplecount)s triples from context %(context)s to %(dumpfile)s in %(elapsed).3f s", + "Dumped %(triplecount)s triples from context %(context)s to %(dumpfile)s (%(elapsed).3f sec)", values): store.get_serialized_file(dump, format="nt", context=context) # just to report the number of dumped triples -- may be unneccesary @@ -1429,7 +1429,7 @@ def relate(self, basefile, otherrepos=[]): """ with util.logtime(self.log.info, - "%(basefile)s: relate OK (%(elapsed).3f s)", + "%(basefile)s: relate OK (%(elapsed).3f sec)", {'basefile': basefile}): # If using the Bulk upload feature, append to the temporary @@ -1439,7 +1439,7 @@ def relate(self, basefile, otherrepos=[]): values = {'basefile': basefile, 'nttemp': nttemp} with util.logtime(self.log.debug, - "%(basefile)s: Added %(triplecount)s triples to %(nttemp)s in %(elapsed).3f s", + "%(basefile)s: Added %(triplecount)s triples to %(nttemp)s (%(elapsed).3f sec)", values): data = open(self.store.distilled_path(basefile), "rb").read() g = Graph().parse(data=data) @@ -1475,7 +1475,7 @@ def relate_triples(self, basefile): """ ts = self._get_triplestore() # init self._triplestore with util.logtime(self.log.debug, - "%(basefile)s: Added %(rdffile)s to context %(context)s in %(elapsed).3f sec", + "%(basefile)s: Added %(rdffile)s to context %(context)s (%(elapsed).3f sec)", {'basefile': basefile, 'context': self.dataset_uri(), 'dataset': self.dataset_uri(), @@ -1509,7 +1509,7 @@ def relate_dependencies(self, basefile, repos=[]): values = {'basefile': basefile, 'deps': 0} with util.logtime(self.log.debug, - "%(basefile)s: Registered %(deps)s dependencies in %(elapsed).3f sec", + "%(basefile)s: Registered %(deps)s dependencies (%(elapsed).3f sec)", values): with self.store.open_distilled(basefile) as fp: g = Graph().parse(fp, format="xml") @@ -1569,7 +1569,7 @@ def relate_fulltext(self, basefile, repos=None): 'resources': 0, 'words': 0} with util.logtime(self.log.debug, - "%(basefile)s: Added %(resources)s resources (%(words)s words) to fulltext index in %(elapsed).3f s", values): + "%(basefile)s: Added %(resources)s resources (%(words)s words) to fulltext index (%(elapsed).3f sec)", values): if repos is None: repos = [] indexer = self._get_fulltext_indexer(repos) @@ -1848,7 +1848,7 @@ def generate(self, basefile, otherrepos=[]): if (self.config.force or (not util.outfile_is_newer(dependencies, self.store.annotation_path(basefile)))): with util.logtime(self.log.debug, - "%(basefile)s: prep_annotation_file in %(elapsed).3f sec", + "%(basefile)s: prep_annotation_file (%(elapsed).3f sec)", {'basefile': basefile}): # annotation_file should be the same as annotations above? annotation_file = self.prep_annotation_file(basefile) @@ -1859,7 +1859,7 @@ def generate(self, basefile, otherrepos=[]): params['annotationfile'] = annotation_file with util.logtime(self.log.debug, - "%(basefile)s: transform in %(elapsed).3f", + "%(basefile)s: transform (%(elapsed).3f sec)", {'basefile': basefile}): conffile = os.path.abspath( os.sep.join([self.config.datadir, 'rsrc', 'resources.xml'])) @@ -2071,7 +2071,7 @@ def toc(self, otherrepos=[]): params = {} with util.logtime(self.log.debug, - "toc: selected %(rowcount)s rows in %(elapsed).3f s", + "toc: selected %(rowcount)s rows (%(elapsed).3f sec)", params): data = self.faceted_data() params['rowcount'] = len(data) @@ -2212,7 +2212,7 @@ def toc_select_for_pages(self, data, pagesets, facets): if page.linktext == key: keyfunc = functools.partial(facet.key, binding=binding, - resource_graph=None) + resource_graph=self.commondata) s = sorted(documents[key], key=keyfunc, reverse=facet.key_descending) diff --git a/ferenda/elements/elements.py b/ferenda/elements/elements.py index 4c72c067..43fedd6f 100644 --- a/ferenda/elements/elements.py +++ b/ferenda/elements/elements.py @@ -235,12 +235,13 @@ def as_xhtml(self, uri=None): # create extra attributes depending on circumstances if hasattr(self,'uri') and self.uri: attrs['about'] = self.uri - + + if hasattr(self,'uri') and self.uri and hasattr(self,'meta') and self.meta: assert isinstance(self.meta,Graph), "self.meta is %r, not rdflib.Graph" % type(self.meta) # we use sorted() to get the triples in a predictable - # order (by predicate) - for (s,p,o) in sorted(self.meta, key=itemgetter(1)): + # order (by predicate, then subject, then object) + for (s,p,o) in sorted(self.meta, key=itemgetter(1,0,2)): if s != URIRef(self.uri): continue if p == RDF.type: @@ -250,8 +251,12 @@ def as_xhtml(self, uri=None): attrs['property'] = self.meta.qname(p) attrs['content'] = o.toPython() else: + # FIXME: Is it sane to reverse the order of + # triples in this way? Maybe we should do a + # children.append instead? children.insert(0, self._span(s,p,o,self.meta)) + # for each childen that is a string, make sure it doesn't # contain any XML illegal characters return E(self.tagname, attrs, *children) diff --git a/ferenda/sources/tech/rfc.py b/ferenda/sources/tech/rfc.py index f92a8476..d2e40045 100644 --- a/ferenda/sources/tech/rfc.py +++ b/ferenda/sources/tech/rfc.py @@ -631,18 +631,22 @@ def selector(entry): with self.store.open_distilled(entry.basefile) as fp: graph.parse(data=fp.read()) desc = Describer(graph, entry.id) - return desc.getvalue(self.ns['dcterms'].subject) == category + return desc.getrel(self.ns['dcterms'].subject) == category return selector return [NewsCriteria('all', 'All RFCs'), NewsCriteria('informational', 'Informational RFCs', - selector=selector_for("Informational")), + selector=selector_for("http://localhost:8000/dataset/rfc/informational")), NewsCriteria('bcp', 'Best Current Practice RFCs', - selector=selector_for("Best Current Practice")), + selector=selector_for("http://localhost:8000/dataset/rfc/best-current-practice")), NewsCriteria('experimental', 'Experimental RFCs', - selector=selector_for("Experimental")), + selector=selector_for("http://localhost:8000/dataset/rfc/experimental")), + NewsCriteria('historic', 'Historic RFCs', + selector=selector_for("http://localhost:8000/dataset/rfc/historic")), + NewsCriteria('unknown', 'Unknown', + selector=selector_for("http://localhost:8000/dataset/rfc/unknown")), NewsCriteria('standards', 'Standards Track RFCs', - selector=selector_for("Standards Track"))] + selector=selector_for("http://localhost:8000/dataset/rfc/standard"))] def frontpage_content(self, primary=False): from rdflib import URIRef diff --git a/test/functionalDocExamples.py b/test/functionalDocExamples.py index a081cbbf..e7a3227d 100644 --- a/test/functionalDocExamples.py +++ b/test/functionalDocExamples.py @@ -38,8 +38,10 @@ def _test_pyfile(self, pyfile, workingdir=None, want=True, comparator=None): oldwd = os.getcwd() pycode = compile(util.readfile(pyfile), pyfile, 'exec') os.chdir(workingdir) - result = six.exec_(pycode, globals(), locals()) - os.chdir(oldwd) + try: + result = six.exec_(pycode, globals(), locals()) + finally: + os.chdir(oldwd) # the exec:ed code is expected to set return_value got = locals()['return_value'] if not comparator: @@ -58,7 +60,7 @@ def mask(self, s): re.compile(r"\((\d+.\d+) sec\)"), re.compile(r" INFO ([\w\-]+): downloaded from http"), re.compile(r": downloaded from (http://[\w\.\-/]+)"), - re.compile(r" INFO ([\w\-]+): OK "), + re.compile(r" INFO ([\w\-]+): (parse|relate|generate) OK "), re.compile(r" DEBUG ([\w\-]+): Created "), re.compile(r" INFO Created data/w3c/toc/([\w/]+).html"), re.compile(r": Created ([\w\-\./]+).xhtml"), @@ -85,8 +87,8 @@ def test_internal_mask(self): "[MASKED] w3c INFO [MASKED]: downloaded from [MASKED]\n[MASKED] w3c INFO [MASKED]: downloaded from [MASKED]\n"), ("20:16:44 root INFO w3c download finished in 14.666 sec", "[MASKED] root INFO w3c download finished in [MASKED] sec"), - ("14:45:57 w3c INFO rdfa-core: OK (2.051 sec)", - "[MASKED] w3c INFO [MASKED]: OK ([MASKED] sec)"), + ("14:45:57 w3c INFO rdfa-core: parse OK (2.051 sec)", + "[MASKED] w3c INFO [MASKED]: parse OK ([MASKED] sec)"), ("15:44:50 w3c DEBUG html-rdfa: Starting", "[MASKED] w3c DEBUG [MASKED]: Starting"), ("15:44:48 w3c DEBUG xhtml-rdfa: Created data/w3c/parsed/xhtml-rfa.xhtml",