maybe now all functional and integration tests work?

doc/examples/firststeps.sh

@@ -75,7 +75,7 @@ $ ./ferenda-build.py w3c download rdfa-core --loglevel=CRITICAL
 
 # begin parse
 $ ./ferenda-build.py w3c parse rdfa-core
-13:04:33 w3c INFO rdfa-core: OK (2.033 sec)
+13:04:33 w3c INFO rdfa-core: parse OK (2.033 sec)
 13:04:33 root INFO w3c parse finished in 2.053 sec
 # end parse
 
@@ -100,7 +100,7 @@ $ ./ferenda-build.py w3c parse rdfa-core
 
 # begin parse-force
 $ ./ferenda-build.py w3c parse rdfa-core --force
-13:04:38 w3c INFO rdfa-core: OK (2.024 sec)
+13:04:38 w3c INFO rdfa-core: parse OK (2.024 sec)
 13:04:38 root INFO w3c parse finished in 2.043 sec
 # end parse-force
 
@@ -111,29 +111,37 @@ $ ./ferenda-build.py w3c parse --all --loglevel=DEBUG
 13:04:39 w3c DEBUG ttml1: Starting
 13:04:43 w3c DEBUG ttml1: Created data/w3c/parsed/ttml1.xhtml
 13:04:45 w3c DEBUG ttml1: 12 triples extracted to data/w3c/distilled/ttml1.rdf
-13:04:45 w3c INFO ttml1: OK (5.816 sec)
+13:04:45 w3c INFO ttml1: parse OK (5.816 sec)
 13:04:45 w3c DEBUG touch-events: Starting
 13:04:45 w3c DEBUG touch-events: Created data/w3c/parsed/touch-events.xhtml
 13:04:45 w3c DEBUG touch-events: 8 triples extracted to data/w3c/distilled/touch-events.rdf
-13:04:45 w3c INFO touch-events: OK (0.486 sec)
+13:04:45 w3c INFO touch-events: parse OK (0.486 sec)
 13:04:45 w3c DEBUG geolocation-API: Starting
 13:04:46 w3c DEBUG geolocation-API: Created data/w3c/parsed/geolocation-API.xhtml
 13:04:46 w3c DEBUG geolocation-API: 5 triples extracted to data/w3c/distilled/geolocation-API.rdf
-13:04:46 w3c INFO geolocation-API: OK (0.323 sec)
+13:04:46 w3c INFO geolocation-API: parse OK (0.323 sec)
 13:04:46 root INFO w3c parse finished in 6.662 sec
 # end parse-all
 
 # begin relate-all
 $ ./ferenda-build.py w3c relate --all
 13:04:47 w3c INFO Clearing context http://localhost:8000/dataset/w3c at repository ferenda
-13:04:54 w3c INFO Dumped 34 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt
+13:04:47 w3c INFO xml-entity-names: relate OK (0.416 sec)
+13:04:47 w3c INFO rdfa-core: relate OK (1.429 sec)
+13:04:47 w3c INFO emotionml: relate OK (1.897 sec)
+13:04:47 w3c INFO MathML3: relate OK (2.494 sec)
+13:04:54 w3c INFO Dumped 34 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt (0.123 sec)
 13:04:54 root INFO w3c relate finished in 7.655 sec
 # end relate-all
 
 # begin makeresources
 $ ./ferenda-build.py w3c makeresources
 $ find data/rsrc -print
 data/rsrc
+data/rsrc/api
+data/rsrc/api/common.json
+data/rsrc/api/context.json
+data/rsrc/api/terms.json
 data/rsrc/css
 data/rsrc/css/ferenda.css
 data/rsrc/css/main.css
@@ -148,10 +156,10 @@ data/rsrc/resources.xml
 
 # begin generate-all
 $ ./ferenda-build.py w3c generate --all
-13:04:58 w3c INFO ttml1: OK (2.102 sec)
-13:04:59 w3c INFO touch-events: OK (0.112 sec)
-13:04:59 w3c INFO rdfa-core: OK (0.220 sec)
-13:04:59 w3c INFO geolocation-API: OK (0.100 sec)
+13:04:58 w3c INFO ttml1: generate OK (2.102 sec)
+13:04:59 w3c INFO touch-events: generate OK (0.112 sec)
+13:04:59 w3c INFO rdfa-core: generate OK (0.220 sec)
+13:04:59 w3c INFO geolocation-API: generate OK (0.100 sec)
 13:04:59 root INFO w3c generate finished in 2.547 sec
 # end generate-all
 
@@ -160,6 +168,9 @@ $ ./ferenda-build.py w3c toc
 13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html
 13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html
 13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html
 13:05:02 w3c INFO Created data/w3c/toc/title/a.html
 13:05:02 w3c INFO Created data/w3c/toc/title/c.html
 13:05:02 w3c INFO Created data/w3c/toc/title/r.html
@@ -185,14 +196,17 @@ $ ./ferenda-build.py w3c all
 13:05:07 root INFO w3cstandards.W3CStandards parse finished in 0.010 sec
 13:05:07 root INFO w3cstandards.W3CStandards relate: Nothing to do!
 13:05:07 root INFO w3cstandards.W3CStandards relate finished in 0.005 sec
-13:05:07 w3c INFO ttml1: OK (0.000 sec)
-13:05:07 w3c INFO touch-events: OK (0.000 sec)
-13:05:07 w3c INFO rdfa-core: OK (0.000 sec)
-13:05:07 w3c INFO geolocation-API: OK (0.000 sec)
+13:05:07 w3c INFO ttml1: generate OK (0.000 sec)
+13:05:07 w3c INFO touch-events: generate OK (0.000 sec)
+13:05:07 w3c INFO rdfa-core: generate OK (0.000 sec)
+13:05:07 w3c INFO geolocation-API: generate OK (0.000 sec)
 13:05:07 root INFO w3cstandards.W3CStandards generate finished in 0.006 sec
 13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html
 13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html
 13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2011.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2014.html
 13:05:02 w3c INFO Created data/w3c/toc/title/a.html
 13:05:02 w3c INFO Created data/w3c/toc/title/c.html
 13:05:02 w3c INFO Created data/w3c/toc/title/r.html

doc/examples/rfcs.py

@@ -296,8 +296,8 @@ def facets(self):
 # begin toc_item
     def toc_item(self, binding, row):
         from ferenda.elements import Link
-        return [row['identifier'] + ": ",
-                Link(row['title'], 
+        return [row['dcterms_identifier'] + ": ",
+                Link(row['dcterms_title'], 
                      uri=row['uri'])]
 # end toc_item
 

ferenda/documentrepository.py

@@ -1403,7 +1403,7 @@ def relate_all_teardown(cls, config):
         # relate_all_setup), do the actual bulk upload.
         if os.path.exists(temp):
             with util.logtime(log.info,
-                              "Loaded %(triplecount)s triples to context %(context)s from %(tempfile)s in %(elapsed).3f s",
+                              "Loaded %(triplecount)s triples to context %(context)s from %(tempfile)s (%(elapsed).3f sec)",
                               values):
                 store.add_serialized_file(temp, format="nt", context=context)
                 # just to report the number of dumped triples -- may be unneccesary
@@ -1414,7 +1414,7 @@ def relate_all_teardown(cls, config):
         # same contents as the temp file, but this comes directly from
         # the triplestore
         with util.logtime(log.info,
-                          "Dumped %(triplecount)s triples from context %(context)s to %(dumpfile)s in %(elapsed).3f s",
+                          "Dumped %(triplecount)s triples from context %(context)s to %(dumpfile)s (%(elapsed).3f sec)",
                           values):
             store.get_serialized_file(dump, format="nt", context=context)
             # just to report the number of dumped triples -- may be unneccesary
@@ -1429,7 +1429,7 @@ def relate(self, basefile, otherrepos=[]):
 
         """
         with util.logtime(self.log.info,
-                          "%(basefile)s: relate OK (%(elapsed).3f s)",
+                          "%(basefile)s: relate OK (%(elapsed).3f sec)",
                           {'basefile': basefile}):
 
             # If using the Bulk upload feature, append to the temporary
@@ -1439,7 +1439,7 @@ def relate(self, basefile, otherrepos=[]):
                 values = {'basefile': basefile,
                           'nttemp': nttemp}
                 with util.logtime(self.log.debug,
-                                  "%(basefile)s: Added %(triplecount)s triples to %(nttemp)s in %(elapsed).3f s",
+                                  "%(basefile)s: Added %(triplecount)s triples to %(nttemp)s (%(elapsed).3f sec)",
                                   values):
                     data = open(self.store.distilled_path(basefile), "rb").read()
                     g = Graph().parse(data=data)
@@ -1475,7 +1475,7 @@ def relate_triples(self, basefile):
         """
         ts = self._get_triplestore()  # init self._triplestore
         with util.logtime(self.log.debug,
-                          "%(basefile)s: Added %(rdffile)s to context %(context)s in %(elapsed).3f sec",
+                          "%(basefile)s: Added %(rdffile)s to context %(context)s (%(elapsed).3f sec)",
                           {'basefile': basefile,
                            'context': self.dataset_uri(),
                            'dataset': self.dataset_uri(),
@@ -1509,7 +1509,7 @@ def relate_dependencies(self, basefile, repos=[]):
         values = {'basefile': basefile,
                   'deps': 0}
         with util.logtime(self.log.debug,
-                          "%(basefile)s: Registered %(deps)s dependencies in %(elapsed).3f sec",
+                          "%(basefile)s: Registered %(deps)s dependencies (%(elapsed).3f sec)",
                           values):
             with self.store.open_distilled(basefile) as fp:
                 g = Graph().parse(fp, format="xml")
@@ -1569,7 +1569,7 @@ def relate_fulltext(self, basefile, repos=None):
                   'resources': 0,
                   'words': 0}
         with util.logtime(self.log.debug,
-                          "%(basefile)s: Added %(resources)s resources (%(words)s words) to fulltext index in %(elapsed).3f s", values):
+                          "%(basefile)s: Added %(resources)s resources (%(words)s words) to fulltext index  (%(elapsed).3f sec)", values):
             if repos is None:
                 repos = []
             indexer = self._get_fulltext_indexer(repos)
@@ -1848,7 +1848,7 @@ def generate(self, basefile, otherrepos=[]):
             if (self.config.force or (not
                     util.outfile_is_newer(dependencies, self.store.annotation_path(basefile)))):
                 with util.logtime(self.log.debug,
-                                  "%(basefile)s: prep_annotation_file in %(elapsed).3f sec",
+                                  "%(basefile)s: prep_annotation_file (%(elapsed).3f sec)",
                                   {'basefile': basefile}):
                     # annotation_file should be the same as annotations above?
                     annotation_file = self.prep_annotation_file(basefile)
@@ -1859,7 +1859,7 @@ def generate(self, basefile, otherrepos=[]):
                 params['annotationfile'] = annotation_file
 
             with util.logtime(self.log.debug,
-                              "%(basefile)s: transform in %(elapsed).3f",
+                              "%(basefile)s: transform (%(elapsed).3f sec)",
                               {'basefile': basefile}):
                 conffile = os.path.abspath(
                     os.sep.join([self.config.datadir, 'rsrc', 'resources.xml']))
@@ -2071,7 +2071,7 @@ def toc(self, otherrepos=[]):
 
         params = {}
         with util.logtime(self.log.debug,
-                          "toc: selected %(rowcount)s rows in %(elapsed).3f s",
+                          "toc: selected %(rowcount)s rows (%(elapsed).3f sec)",
                           params):
             data = self.faceted_data()
             params['rowcount'] = len(data)
@@ -2212,7 +2212,7 @@ def toc_select_for_pages(self, data, pagesets, facets):
                     if page.linktext == key:
                         keyfunc = functools.partial(facet.key,
                                                     binding=binding,
-                                                    resource_graph=None)
+                                                    resource_graph=self.commondata)
                         s = sorted(documents[key],
                                    key=keyfunc,
                                    reverse=facet.key_descending)

ferenda/elements/elements.py

@@ -235,12 +235,13 @@ def as_xhtml(self, uri=None):
         # create extra attributes depending on circumstances
         if hasattr(self,'uri') and self.uri:
             attrs['about'] = self.uri
-
+
+
         if hasattr(self,'uri') and self.uri and hasattr(self,'meta') and self.meta:
             assert isinstance(self.meta,Graph), "self.meta is %r, not rdflib.Graph" % type(self.meta)
             # we use sorted() to get the triples in a predictable
-            # order (by predicate)
-            for (s,p,o) in sorted(self.meta, key=itemgetter(1)):
+            # order (by predicate, then subject, then object)
+            for (s,p,o) in sorted(self.meta, key=itemgetter(1,0,2)):
                 if s != URIRef(self.uri):
                     continue
                 if p == RDF.type:
@@ -250,8 +251,12 @@ def as_xhtml(self, uri=None):
                     attrs['property'] = self.meta.qname(p)
                     attrs['content'] = o.toPython()
                 else:
+                    # FIXME: Is it sane to reverse the order of
+                    # triples in this way? Maybe we should do a
+                    # children.append instead?
                     children.insert(0, self._span(s,p,o,self.meta))
 
+
         # for each childen that is a string, make sure it doesn't
         # contain any XML illegal characters
         return E(self.tagname, attrs, *children)

ferenda/sources/tech/rfc.py

@@ -631,18 +631,22 @@ def selector(entry):
                 with self.store.open_distilled(entry.basefile) as fp:
                     graph.parse(data=fp.read())
                 desc = Describer(graph, entry.id)
-                return desc.getvalue(self.ns['dcterms'].subject) == category
+                return desc.getrel(self.ns['dcterms'].subject) == category
             return selector
 
         return [NewsCriteria('all', 'All RFCs'),
                 NewsCriteria('informational', 'Informational RFCs',
-                             selector=selector_for("Informational")),
+                             selector=selector_for("http://localhost:8000/dataset/rfc/informational")),
                 NewsCriteria('bcp', 'Best Current Practice RFCs',
-                             selector=selector_for("Best Current Practice")),
+                             selector=selector_for("http://localhost:8000/dataset/rfc/best-current-practice")),
                 NewsCriteria('experimental', 'Experimental RFCs',
-                             selector=selector_for("Experimental")),
+                             selector=selector_for("http://localhost:8000/dataset/rfc/experimental")),
+                NewsCriteria('historic', 'Historic RFCs',
+                             selector=selector_for("http://localhost:8000/dataset/rfc/historic")),
+                NewsCriteria('unknown', 'Unknown',
+                             selector=selector_for("http://localhost:8000/dataset/rfc/unknown")),
                 NewsCriteria('standards', 'Standards Track RFCs',
-                             selector=selector_for("Standards Track"))]
+                             selector=selector_for("http://localhost:8000/dataset/rfc/standard"))]
 
     def frontpage_content(self, primary=False):
         from rdflib import URIRef

test/functionalDocExamples.py

@@ -38,8 +38,10 @@ def _test_pyfile(self, pyfile, workingdir=None, want=True, comparator=None):
         oldwd = os.getcwd()
         pycode = compile(util.readfile(pyfile), pyfile, 'exec')
         os.chdir(workingdir)
-        result = six.exec_(pycode, globals(), locals())
-        os.chdir(oldwd)
+        try:
+            result = six.exec_(pycode, globals(), locals())
+        finally:
+            os.chdir(oldwd)
         # the exec:ed code is expected to set return_value
         got = locals()['return_value']
         if not comparator:
@@ -58,7 +60,7 @@ def mask(self, s):
             re.compile(r"\((\d+.\d+) sec\)"),
             re.compile(r" INFO ([\w\-]+): downloaded from http"),
             re.compile(r": downloaded from (http://[\w\.\-/]+)"),
-            re.compile(r" INFO ([\w\-]+): OK "),
+            re.compile(r" INFO ([\w\-]+): (parse|relate|generate) OK "),
             re.compile(r" DEBUG ([\w\-]+): Created "),
             re.compile(r" INFO Created data/w3c/toc/([\w/]+).html"),
             re.compile(r": Created ([\w\-\./]+).xhtml"),
@@ -85,8 +87,8 @@ def test_internal_mask(self):
                  "[MASKED] w3c INFO [MASKED]: downloaded from [MASKED]\n[MASKED] w3c INFO [MASKED]: downloaded from [MASKED]\n"),
                 ("20:16:44 root INFO w3c download finished in 14.666 sec",
                  "[MASKED] root INFO w3c download finished in [MASKED] sec"),
-                ("14:45:57 w3c INFO rdfa-core: OK (2.051 sec)",
-                 "[MASKED] w3c INFO [MASKED]: OK ([MASKED] sec)"),
+                ("14:45:57 w3c INFO rdfa-core: parse OK (2.051 sec)",
+                 "[MASKED] w3c INFO [MASKED]: parse OK ([MASKED] sec)"),
                 ("15:44:50 w3c DEBUG html-rdfa: Starting",
                  "[MASKED] w3c DEBUG [MASKED]: Starting"),
                 ("15:44:48 w3c DEBUG xhtml-rdfa: Created data/w3c/parsed/xhtml-rfa.xhtml",