Merge pull request #3 from staffanm/feature/coverage

Feature/coverage -- make test coverage figures less embarrasing

.gitignore

@@ -1,8 +1,8 @@
 *~
 *.pyc
 .DS_Store
+.ropeproject
 /.coverage
-/.ropeproject
 /.tox
 /dist/
 /ferenda.egg-info/

.travis.yml

@@ -1,19 +1,17 @@
 language: python
 python:
-  - "2.7"
   - "2.6"
+  - "2.7"
   - "3.2"
   - "3.3"
 before_install:
-  - sudo apt-get install -qq python-simpleparse antiword
+  - sudo apt-get install -qq python-simpleparse antiword poppler-utils
 services:
   - elasticsearch
 install:
   - if [[ ${TRAVIS_PYTHON_VERSION%%.*} == '2' ]]; then pip install --use-mirrors -r requirements.py2.txt; fi
   - if [[ ${TRAVIS_PYTHON_VERSION%%.*} == '3' ]]; then LANG=en_US.UTF-8 pip install --use-mirrors -r requirements.py3.txt; fi
   - pip install coveralls --use-mirrors
-env:
-  - SKIP_FUSEKI_TESTS=1 SKIP_SESAME_TESTS=1 SKIP_SLEEPYCAT_TESTS=1
 script:
   - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then PYTHONWARNINGS=i coverage run --include "ferenda/*py" --omit "ferenda/thirdparty/*" -m unittest2 discover test; fi
   - if [[ $TRAVIS_PYTHON_VERSION != '2.6' ]]; then PYTHONWARNINGS=i coverage run --include "ferenda/*py" --omit "ferenda/thirdparty/*" -m unittest discover test; fi

doc/advanced.rst

@@ -36,10 +36,12 @@ the ``subrepos`` class property.
    :end-before: # end composite
 
 The CompositeRepository docrepo then acts as a proxy for all of your
-specialized repositories::
+specialized repositories:
 
 .. literalinclude:: examples/composite-repository.sh
-
+   :start-after: # begin example
+   :end-before: # end example
+
 Note that ``patents.XMLPatents`` and the other subrepos are never
 registered in ferenda.ini``. They're just called behind-the-scenes by
 ``patents.CompositePatents``.

doc/examples/citationparsing-parsers.py

@@ -6,7 +6,8 @@
 from bs4 import BeautifulSoup
 
 doc = Mock()
-doc.body = elements_from_soup(BeautifulSoup(open("doc/examples/citationparsing-before.xhtml").read()).body)
+filedir = os.path.dirname(__file__) 
+doc.body = elements_from_soup(BeautifulSoup(open(filedir+"/../doc/examples/citationparsing-before.xhtml").read()).body)
 
 # begin
 from pyparsing import Word, nums

doc/examples/composite-repository.sh

@@ -1,3 +1,7 @@
+$ ferenda-setup patents
+$ cd patents
+$ mv ../patents.py .
+# begin example
 $ ./ferenda-build.py patents.CompositePatents enable
 # calls download() for all subrepos
 $ ./ferenda-build.py pat download 
@@ -7,3 +11,4 @@ $ ./ferenda-build.py pat parse 5723765
 # uses the pat/parsed/5723765 data. From here on, we're just like any
 # other docrepo.
 $ ./ferenda-build.py pat generate 5723765 
+# end example

doc/examples/firststeps-api.py

@@ -3,6 +3,7 @@
 
 # firststeps-api.py
 import sys
+import shutil
 sys.path.append("doc/examples") # to find w3cstandards.py
 
 # begin download-status
@@ -13,7 +14,7 @@
 # or use repo.get_status() to get all status information in a nested dict
 # end download-status
 
-# make sure the basid we use for examples is available
+# make sure the basefile we use for examples is available
 repo.download("rdfa-core")
 
 # begin parse-force
@@ -49,5 +50,5 @@
 repo.news()
 manager.frontpage([repo])
 # end final-commands
-
+shutil.rmtree(repo.config.datadir)
 return_value = True

doc/examples/firststeps.sh

@@ -17,58 +17,66 @@ $ mv ../w3cstandards.py .
 
 # begin enable
 $ ./ferenda-build.py w3cstandards.W3CStandards enable
-12:22:18 root INFO Enabled class w3cstandards.W3CStandards (alias 'w3c')
+13:04:16 root INFO Enabled class w3cstandards.W3CStandards (alias 'w3c')
 # end enable
 
 # begin status-example
 $ ./ferenda-build.py w3cstandards.W3CStandards status # verbose
-12:22:20 root INFO w3cstandards.W3CStandards status finished in 0.004 sec
+13:04:17 root INFO w3cstandards.W3CStandards status finished in 0.004 sec
 Status for document repository 'w3c' (w3cstandards.W3CStandards)
  download: None.
  parse: None.
  generated: None.
 
 $ ./ferenda-build.py w3c status # terse, exactly the same result
 # end status-example
-12:22:20 root INFO w3c status finished in 0.004 sec
+13:04:17 root INFO w3c status finished in 0.004 sec
 Status for document repository 'w3c' (w3cstandards.W3CStandards)
  download: None.
  parse: None.
  generated: None.
 
+
 # begin download
 $ ./ferenda-build.py w3c download 
-20:16:42 w3c INFO Downloading max 3 documents
-20:16:43 w3c INFO rdfa-core: downloaded from http://www.w3.org/TR/2013/REC-rdfa-core-20130822/
-20:16:44 w3c INFO xhtml-rdfa: downloaded from http://www.w3.org/TR/2013/REC-xhtml-rdfa-20130822/
-20:16:44 w3c INFO html-rdfa: downloaded from http://www.w3.org/TR/2013/REC-html-rdfa-20130822/
+13:04:21 w3c INFO Downloading max 3 documents
+13:04:22 w3c INFO geolocation-API: downloaded from http://www.w3.org/TR/2013/REC-geolocation-API-20131024/
+13:04:23 w3c INFO touch-events: downloaded from http://www.w3.org/TR/2013/REC-touch-events-20131010/
+13:04:25 w3c INFO ttml1: downloaded from http://www.w3.org/TR/2013/REC-ttml1-20130924/
 # and so on...
 # end download
-20:16:44 root INFO w3c download finished in 4.666 sec
-$ 
+13:04:25 root INFO w3c download finished in 5.958 sec
+
 # begin list-downloaded
 $ ls -1 data/w3c/downloaded
-html-rdfa.html
-html-rdfa.html.etag
-rdfa-core.html
-rdfa-core.html.etag
-xhtml-rdfa.html
-xhtml-rdfa.html.etag
+geolocation-API.html
+geolocation-API.html.etag
+touch-events.html
+touch-events.html.etag
+ttml1.html
+ttml1.html.etag
 # end list-downloaded
 
 # begin status
 $ ./ferenda-build.py w3c status
-20:18:21 root INFO w3c status finished in 0.013 sec
+13:04:26 root INFO w3c status finished in 0.014 sec
 Status for document repository 'w3c' (w3cstandards.W3CStandards)
- download: xhtml-rdfa, rdfa-core, html-rdfa.
- parse: None. Todo: xhtml-rdfa, rdfa-core, html-rdfa.
+ download: ttml1, touch-events, geolocation-API.
+ parse: None. Todo: ttml1, touch-events, geolocation-API.
  generated: None.
 # end status
 
+# make sure the basefile we use for examples is available. To match
+# logging output, it should not be one of the basefiles downloaded
+# above
+# begin single-download
+$ ./ferenda-build.py w3c download rdfa-core --loglevel=CRITICAL
+# end single-download
+
 # begin parse
 $ ./ferenda-build.py w3c parse rdfa-core
-14:45:57 w3c INFO rdfa-core: OK (2.051 sec)
-14:45:57 root INFO w3c parse finished in 2.068 sec
+13:04:33 w3c INFO rdfa-core: OK (2.033 sec)
+13:04:33 root INFO w3c parse finished in 2.053 sec
 # end parse
 
 # begin list-parsed
@@ -78,43 +86,47 @@ rdfa-core.xhtml
 
 # begin status-2
 $ ./ferenda-build.py w3c status
-14:59:56 root INFO w3c status finished in 0.014 sec
+13:04:34 root INFO w3c status finished in 0.013 sec
 Status for document repository 'w3c' (w3cstandards.W3CStandards)
- download: xhtml-rdfa, rdfa-core, html-rdfa.
- parse: rdfa-core. Todo: xhtml-rdfa, html-rdfa.
+ download: ttml1, touch-events, rdfa-core... (1 more)
+ parse: rdfa-core. Todo: ttml1, touch-events, geolocation-API.
  generated: None. Todo: rdfa-core.
 # end status-2
 
 # begin parse-again
 $ ./ferenda-build.py w3c parse rdfa-core
-10:06:15 root INFO w3c parse finished in 0.014 sec
+13:04:35 root INFO w3c parse finished in 0.016 sec
 # end parse-again
 
 # begin parse-force
 $ ./ferenda-build.py w3c parse rdfa-core --force
-14:45:57 w3c INFO rdfa-core: OK (2.051 sec)
-14:45:57 root INFO w3c parse finished in 2.068 sec
+13:04:38 w3c INFO rdfa-core: OK (2.024 sec)
+13:04:38 root INFO w3c parse finished in 2.043 sec
 # end parse-force
 
 # begin parse-all
 $ ./ferenda-build.py w3c parse --all --loglevel=DEBUG
-15:44:48 w3c DEBUG xhtml-rdfa: Starting
-15:44:48 w3c DEBUG xhtml-rdfa: Created data/w3c/parsed/xhtml-rdfa.xhtml
-15:44:48 w3c DEBUG xhtml-rdfa: 5 triples extracted to data/w3c/distilled/xhtml-rdfa.rdf
-15:44:48 w3c INFO xhtml-rdfa: OK (0.567 sec)
-15:44:48 w3c DEBUG rdfa-core: Skipped
-15:44:50 w3c DEBUG html-rdfa: Starting
-15:44:51 w3c DEBUG html-rdfa: Created data/w3c/parsed/html-rdfa.xhtml
-15:44:51 w3c DEBUG html-rdfa: 11 triples extracted to data/w3c/distilled/html-rdfa.rdf
-15:44:51 w3c INFO html-rdfa: OK (0.552 sec)
-15:44:51 root INFO w3c parse finished in 3.128 sec
+13:04:39 w3c DEBUG ttml1: Starting
+13:04:43 w3c DEBUG ttml1: Created data/w3c/parsed/ttml1.xhtml
+13:04:45 w3c DEBUG ttml1: 12 triples extracted to data/w3c/distilled/ttml1.rdf
+13:04:45 w3c INFO ttml1: OK (5.816 sec)
+13:04:45 w3c DEBUG touch-events: Starting
+13:04:45 w3c DEBUG touch-events: Created data/w3c/parsed/touch-events.xhtml
+13:04:45 w3c DEBUG touch-events: 8 triples extracted to data/w3c/distilled/touch-events.rdf
+13:04:45 w3c INFO touch-events: OK (0.486 sec)
+13:04:45 w3c DEBUG rdfa-core: Skipped
+13:04:45 w3c DEBUG geolocation-API: Starting
+13:04:46 w3c DEBUG geolocation-API: Created data/w3c/parsed/geolocation-API.xhtml
+13:04:46 w3c DEBUG geolocation-API: 5 triples extracted to data/w3c/distilled/geolocation-API.rdf
+13:04:46 w3c INFO geolocation-API: OK (0.323 sec)
+13:04:46 root INFO w3c parse finished in 6.662 sec
 # end parse-all
 
 # begin relate-all
 $ ./ferenda-build.py w3c relate --all
-15:21:05 w3c INFO Clearing context http://localhost:8000/dataset/w3c at repository ferenda
-15:21:10 w3c INFO Dumped 25 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt
-15:21:10 root INFO w3c relate finished in 5.215 sec
+13:04:47 w3c INFO Clearing context http://localhost:8000/dataset/w3c at repository ferenda
+13:04:54 w3c INFO Dumped 34 triples from context http://localhost:8000/dataset/w3c to data/w3c/distilled/dump.nt
+13:04:54 root INFO w3c relate finished in 7.655 sec
 # end relate-all
 
 # begin makeresources
@@ -134,25 +146,27 @@ data/rsrc/resources.xml
 
 # begin generate-all
 $ ./ferenda-build.py w3c generate --all
-15:26:37 w3c INFO xhtml-rdfa OK (1.628 sec)
-15:26:37 w3c INFO rdfa-core OK (0.227 sec)
-15:26:37 w3c INFO html-rdfa OK (0.105 sec)
-15:26:37 root INFO w3c generate finished in 1.973 sec
+13:04:58 w3c INFO ttml1: OK (2.102 sec)
+13:04:59 w3c INFO touch-events: OK (0.112 sec)
+13:04:59 w3c INFO rdfa-core: OK (0.220 sec)
+13:04:59 w3c INFO geolocation-API: OK (0.100 sec)
+13:04:59 root INFO w3c generate finished in 2.547 sec
 # end generate-all
 
 # begin final-commands
 $ ./ferenda-build.py w3c toc
-16:11:39 w3c INFO Created data/w3c/toc/issued/2013.html
-16:11:39 w3c INFO Created data/w3c/toc/title/h.html
-16:11:39 w3c INFO Created data/w3c/toc/title/r.html
-16:11:39 w3c INFO Created data/w3c/toc/title/x.html
-16:11:39 w3c INFO Created data/w3c/toc/index.html
-16:11:39 root INFO w3c toc finished in 1.658 sec
+13:05:01 w3c INFO Created data/w3c/toc/issued/2004.html
+13:05:01 w3c INFO Created data/w3c/toc/issued/2013.html
+13:05:01 w3c INFO Created data/w3c/toc/title/g.html
+13:05:02 w3c INFO Created data/w3c/toc/title/r.html
+13:05:02 w3c INFO Created data/w3c/toc/title/t.html
+13:05:02 w3c INFO Created data/w3c/toc/index.html
+13:05:02 root INFO w3c toc finished in 1.739 sec
 $ ./ferenda-build.py w3c news
-16:30:51 w3c INFO feed main: 3 entries
-16:30:51 root INFO w3c news finished in 0.067 sec
+13:05:03 w3c INFO feed main: 4 entries
+13:05:03 root INFO w3c news finished in 0.086 sec
 $ ./ferenda-build.py w3c frontpage
-15:28:59 root INFO frontpage: wrote data/index.html (0.016 sec)
+13:05:04 root INFO frontpage: wrote data/index.html (0.017 sec)
 # end final-commands
 
 # begin runserver
@@ -162,22 +176,27 @@ $ ./ferenda-build.py w3c frontpage
 
 # begin all
 $ ./ferenda-build.py w3c all
-10:45:05 w3c INFO Downloading max 3 documents
-10:45:05 root INFO w3cstandards.W3CStandards download finished in 0.977 sec
-10:45:05 root INFO w3cstandards.W3CStandards parse finished in 0.009 sec
-10:45:05 root INFO w3cstandards.W3CStandards relate: Nothing to do!
-10:45:05 root INFO w3cstandards.W3CStandards relate finished in 0.004 sec
-10:45:05 w3c INFO xhtml-rdfa OK (0.000 sec)
-10:45:05 w3c INFO rdfa-core OK (0.000 sec)
-10:45:05 w3c INFO html-rdfa OK (0.000 sec)
-10:45:05 root INFO w3cstandards.W3CStandards generate finished in 0.006 sec
-10:45:07 w3c INFO Created data/w3c/toc/issued/2013.html
-10:45:07 w3c INFO Created data/w3c/toc/title/h.html
-10:45:07 w3c INFO Created data/w3c/toc/title/r.html
-10:45:07 w3c INFO Created data/w3c/toc/title/x.html
-10:45:07 w3c INFO Created data/w3c/toc/index.html
-10:45:07 root INFO w3cstandards.W3CStandards toc finished in 1.655 sec
-10:45:07 w3c INFO feed main: 3 entries
-10:45:07 root INFO w3cstandards.W3CStandards news finished in 0.045 sec
-10:45:07 root INFO frontpage: wrote data/index.html (0.012 sec)
+13:05:07 w3c INFO Downloading max 3 documents
+13:05:07 root INFO w3cstandards.W3CStandards download finished in 2.476 sec
+13:05:07 root INFO w3cstandards.W3CStandards parse finished in 0.010 sec
+13:05:07 root INFO w3cstandards.W3CStandards relate: Nothing to do!
+13:05:07 root INFO w3cstandards.W3CStandards relate finished in 0.005 sec
+13:05:07 w3c INFO ttml1: OK (0.000 sec)
+13:05:07 w3c INFO touch-events: OK (0.000 sec)
+13:05:07 w3c INFO rdfa-core: OK (0.000 sec)
+13:05:07 w3c INFO geolocation-API: OK (0.000 sec)
+13:05:07 root INFO w3cstandards.W3CStandards generate finished in 0.006 sec
+13:05:09 w3c INFO Created data/w3c/toc/issued/2004.html
+13:05:09 w3c INFO Created data/w3c/toc/issued/2013.html
+13:05:09 w3c INFO Created data/w3c/toc/title/g.html
+13:05:09 w3c INFO Created data/w3c/toc/title/r.html
+13:05:09 w3c INFO Created data/w3c/toc/title/t.html
+13:05:09 w3c INFO Created data/w3c/toc/index.html
+13:05:09 root INFO w3cstandards.W3CStandards toc finished in 1.705 sec
+13:05:09 w3c INFO feed main: 4 entries
+13:05:09 root INFO w3cstandards.W3CStandards news finished in 0.057 sec
+13:05:09 root INFO frontpage: wrote data/index.html (0.013 sec)
 # end all
+
+$ cd ..
+$ rm -r netstandards

doc/examples/intro-example.py

@@ -1,5 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
+import shutil, os
+if os.path.exists("netstandards"):
+    shutil.rmtree("netstandards")
+
+# begin example
 from ferenda.sources.tech import RFC, W3Standards
 from ferenda.manager import makeresources, frontpage, runserver, setup_logger
 from ferenda.errors import DocumentRemovedError, ParseError, FSMStateError
@@ -55,4 +60,8 @@
 
 # Start WSGI app at http://localhost:8000/ with navigation,
 # document viewing, search and API
-runserver(docrepos, port=8000, documentroot="netstandards/exampledata")
+# runserver(docrepos, port=8000, documentroot="netstandards/exampledata")
+
+# end example
+shutil.rmtree("netstandards")
+return_value = True

doc/examples/intro-example.sh

@@ -1,8 +1,8 @@
-$ ./ferenda-setup.py netstandards
+$ ferenda-setup netstandards
 $ cd netstandards
 $ ./ferenda-build.py ferenda.sources.tech.RFC enable
 $ ./ferenda-build.py ferenda.sources.tech.W3Standards enable
-$ ./ferenda-build.py all all --downloadmax=5
-$ ./ferenda-build.py all runserver &
-$ open http://localhost:8000/
+$ ./ferenda-build.py all all --downloadmax=50
+# $ ./ferenda-build.py all runserver &
+# $ open http://localhost:8000/
 

doc/examples/keyconcepts-file.py

@@ -5,6 +5,7 @@ class Test(object):
     store = DocumentStore(datadir="data/base")
 
     def do(self, basefile):
+        util.ensure_dir(self.store.downloaded_path(basefile))
 # begin path
         path = self.store.downloaded_path(basefile)
         with open(path, mode="wb") as fp: