Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Test running using single command (nltk/test/runtests.py)

  • Loading branch information...
commit ee1416e72de74f5c64600f728d4539b09917d19e 1 parent 3d1e0c7
@kmike kmike authored
View
7 nltk/align.py
@@ -301,11 +301,10 @@ class IBMModel1(object):
on the translation probabilities from Stage 1.
.. doctest::
- :options: +SKIP
- >>> from nltk.corpus import comtrans
- >>> from nltk.align import IBMModel1
- >>> ibm1 = IBMModel1(comtrans.aligned_sents())
+ >> from nltk.corpus import comtrans
+ >> from nltk.align import IBMModel1
+ >> ibm1 = IBMModel1(comtrans.aligned_sents())
:param aligned_sents: The parallel text ``corpus.Iterable`` containing
AlignedSent instances of aligned sentence pairs from the corpus.
View
28 nltk/test/align.doctest
@@ -72,36 +72,36 @@ EM for IBM Model 1
Here is an example from Kohn, 2010:
- >>> from nltk.align import IBMModel1
+ >>> from nltk.align import IBMModel1 # doctest +SKIP
>>> corpus = [AlignedSent(['the', 'house'], ['das', 'Haus']),
... AlignedSent(['the', 'book'], ['das', 'Buch']),
- ... AlignedSent(['a', 'book'], ['ein', 'Buch'])]
- >>> em_ibm1 = IBMModel1(corpus, 1e-3)
- >>> print round(em_ibm1.probabilities['the', 'das'], 1)
+ ... AlignedSent(['a', 'book'], ['ein', 'Buch'])] # doctest +SKIP
+ >>> em_ibm1 = IBMModel1(corpus, 1e-3) # doctest +SKIP
+ >>> print round(em_ibm1.probabilities['the', 'das'], 1) # doctest +SKIP
1.0
- >>> print round(em_ibm1.probabilities['book', 'das'], 1)
+ >>> print round(em_ibm1.probabilities['book', 'das'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['house', 'das'], 1)
+ >>> print round(em_ibm1.probabilities['house', 'das'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['the', 'Buch'], 1)
+ >>> print round(em_ibm1.probabilities['the', 'Buch'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['book', 'Buch'], 1)
+ >>> print round(em_ibm1.probabilities['book', 'Buch'], 1) # doctest +SKIP
1.0
- >>> print round(em_ibm1.probabilities['a', 'Buch'], 1)
+ >>> print round(em_ibm1.probabilities['a', 'Buch'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['book', 'ein'], 1)
+ >>> print round(em_ibm1.probabilities['book', 'ein'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['a', 'ein'], 1)
+ >>> print round(em_ibm1.probabilities['a', 'ein'], 1) # doctest +SKIP
1.0
- >>> print round(em_ibm1.probabilities['the', 'Haus'], 1)
+ >>> print round(em_ibm1.probabilities['the', 'Haus'], 1) # doctest +SKIP
0.0
- >>> print round(em_ibm1.probabilities['house', 'Haus'], 1)
+ >>> print round(em_ibm1.probabilities['house', 'Haus'], 1) # doctest +SKIP
1.0
Get the alignments:
- >>> em_ibm1.aligned() # doctest: +NORMALIZE_WHITESPACE
+ >>> em_ibm1.aligned() # doctest: +SKIP
[AlignedSent(['the', 'house'], ['das', 'Haus'],
Alignment([(0, 0), (1, 1)])),
AlignedSent(['the', 'book'], ['das', 'Buch'],
View
1,188 nltk/test/coverage.py
@@ -1,1188 +0,0 @@
-#!/usr/bin/python
-#
-# Perforce Defect Tracking Integration Project
-# <http://www.ravenbrook.com/project/p4dti/>
-#
-# COVERAGE.PY -- COVERAGE TESTING
-#
-# Gareth Rees, Ravenbrook Limited, 2001-12-04
-# Ned Batchelder, 2004-12-12
-# http://nedbatchelder.com/code/modules/coverage.html
-#
-#
-# 1. INTRODUCTION
-#
-# This module provides coverage testing for Python code.
-#
-# The intended readership is all Python developers.
-#
-# This document is not confidential.
-#
-# See [GDR 2001-12-04a] for the command-line interface, programmatic
-# interface and limitations. See [GDR 2001-12-04b] for requirements and
-# design.
-
-r"""Usage:
-
-coverage.py -x [-p] MODULE.py [ARG1 ARG2 ...]
- Execute module, passing the given command-line arguments, collecting
- coverage data. With the -p option, write to a temporary file containing
- the machine name and process ID.
-
-coverage.py -e
- Erase collected coverage data.
-
-coverage.py -c
- Collect data from multiple coverage files (as created by -p option above)
- and store it into a single file representing the union of the coverage.
-
-coverage.py -r [-m] [-o dir1,dir2,...] FILE1 FILE2 ...
- Report on the statement coverage for the given files. With the -m
- option, show line numbers of the statements that weren't executed.
-
-coverage.py -a [-d dir] [-o dir1,dir2,...] FILE1 FILE2 ...
- Make annotated copies of the given files, marking statements that
- are executed with > and statements that are missed with !. With
- the -d option, make the copies in that directory. Without the -d
- option, make each copy in the same directory as the original.
-
--o dir,dir2,...
- Omit reporting or annotating files when their filename path starts with
- a directory listed in the omit list.
- e.g. python coverage.py -i -r -o c:\python23,lib\enthought\traits
-
-Coverage data is saved in the file .coverage by default. Set the
-COVERAGE_FILE environment variable to save it somewhere else."""
-
-__docformat__ = 'plaintext en'
-__version__ = "2.77.20070729" # see detailed history at the end of this file.
-
-import compiler
-import compiler.visitor
-import glob
-import os
-import re
-import string
-import symbol
-import sys
-import threading
-import token
-import types
-from socket import gethostname
-
-# Python version compatibility
-try:
- strclass = basestring # new to 2.3
-except:
- strclass = str
-
-# 2. IMPLEMENTATION
-#
-# This uses the "singleton" pattern.
-#
-# The word "morf" means a module object (from which the source file can
-# be deduced by suitable manipulation of the __file__ attribute) or a
-# filename.
-#
-# When we generate a coverage report we have to canonicalize every
-# filename in the coverage dictionary just in case it refers to the
-# module we are reporting on. It seems a shame to throw away this
-# information so the data in the coverage dictionary is transferred to
-# the 'cexecuted' dictionary under the canonical filenames.
-#
-# The coverage dictionary is called "c" and the trace function "t". The
-# reason for these short names is that Python looks up variables by name
-# at runtime and so execution time depends on the length of variables!
-# In the bottleneck of this application it's appropriate to abbreviate
-# names to increase speed.
-
-class DefInfo:
- FUNC = 'func'
- CLASS = 'class'
- def __init__(self, node, defstart, codestart, end, coverage=None):
- self.typ = self.type_for(node)
- self.name = node.name
- self.defstart = defstart
- self.codestart = codestart
- self.end = end
- self.coverage = coverage
-
- def __repr__(self):
- return ('DefInfo(%s, %s, %s, %s, %s, %s)' %
- (self.typ, self.name, self.defstart, self.codestart,
- self.end, self.coverage))
-
- @staticmethod
- def type_for(node):
- if isinstance(node, compiler.ast.Function):
- return DefInfo.FUNC
- else:
- return DefInfo.CLASS
-
-class StatementFindingAstVisitor(compiler.visitor.ASTVisitor):
- """ A visitor for a parsed Abstract Syntax Tree which finds executable
- statements.
- """
- def __init__(self, statements, excluded, suite_spots, definfo):
- compiler.visitor.ASTVisitor.__init__(self)
- self.statements = statements
- self.excluded = excluded
- self.suite_spots = suite_spots
- self.excluding_suite = 0
- self.definfo = definfo
-
- def doRecursive(self, node):
- for n in node.getChildNodes():
- self.dispatch(n)
-
- visitStmt = visitModule = doRecursive
-
- def doCode(self, node):
- if hasattr(node, 'decorators') and node.decorators:
- self.dispatch(node.decorators)
- self.recordAndDispatch(node.code)
- else:
- self.doSuite(node, node.code)
-
- defstart = self.getFirstLine(node)
- codestart = self.getFirstLine(node.code)
- end = self.getLastLine(node.code)
- self.definfo.append(DefInfo(node, defstart, codestart, end))
- # When we exit a block, build up dotted names.
- for i in range(len(self.definfo)-2, -1, -1):
- if self.definfo[i].defstart < defstart: break
- self.definfo[i].name = '%s.%s' % (node.name, self.definfo[i].name)
- # When we exit a func block, hide any contained objects.
- if len(self.definfo)>1 and self.definfo[-1].typ == DefInfo.FUNC:
- del self.definfo[i+1:-1]
-
- visitFunction = visitClass = doCode
-
- def getFirstLine(self, node):
- # Find the first line in the tree node.
- lineno = node.lineno
- for n in node.getChildNodes():
- f = self.getFirstLine(n)
- if lineno and f:
- lineno = min(lineno, f)
- else:
- lineno = lineno or f
- return lineno
-
- def getLastLine(self, node):
- # Find the first line in the tree node.
- lineno = node.lineno
- for n in node.getChildNodes():
- lineno = max(lineno, self.getLastLine(n))
- return lineno
-
- def doStatement(self, node):
- self.recordLine(self.getFirstLine(node))
-
- visitAssert = visitAssign = visitAssTuple = visitPrint = \
- visitPrintnl = visitRaise = visitSubscript = visitDecorators = \
- doStatement
-
- def visitPass(self, node):
- # Pass statements have weird interactions with docstrings. If this
- # pass statement is part of one of those pairs, claim that the statement
- # is on the later of the two lines.
- l = node.lineno
- if l:
- lines = self.suite_spots.get(l, [l,l])
- self.statements[lines[1]] = 1
-
- def visitDiscard(self, node):
- # Discard nodes are statements that execute an expression, but then
- # discard the results. This includes function calls, so we can't
- # ignore them all. But if the expression is a constant, the statement
- # won't be "executed", so don't count it now.
- if node.expr.__class__.__name__ != 'Const':
- self.doStatement(node)
-
- def recordNodeLine(self, node):
- # Stmt nodes often have None, but shouldn't claim the first line of
- # their children (because the first child might be an ignorable line
- # like "global a").
- if node.__class__.__name__ != 'Stmt':
- return self.recordLine(self.getFirstLine(node))
- else:
- return 0
-
- def recordLine(self, lineno):
- # Returns a bool, whether the line is included or excluded.
- if lineno:
- # Multi-line tests introducing suites have to get charged to their
- # keyword.
- if lineno in self.suite_spots:
- lineno = self.suite_spots[lineno][0]
- # If we're inside an excluded suite, record that this line was
- # excluded.
- if self.excluding_suite:
- self.excluded[lineno] = 1
- return 0
- # If this line is excluded, or suite_spots maps this line to
- # another line that is exlcuded, then we're excluded.
- elif lineno in self.excluded or \
- lineno in self.suite_spots and \
- self.suite_spots[lineno][1] in self.excluded:
- return 0
- # Otherwise, this is an executable line.
- else:
- self.statements[lineno] = 1
- return 1
- return 0
-
- default = recordNodeLine
-
- def recordAndDispatch(self, node):
- self.recordNodeLine(node)
- self.dispatch(node)
-
- def doSuite(self, intro, body, exclude=0):
- exsuite = self.excluding_suite
- if exclude or (intro and not self.recordNodeLine(intro)):
- self.excluding_suite = 1
- self.recordAndDispatch(body)
- self.excluding_suite = exsuite
-
- def doPlainWordSuite(self, prevsuite, suite):
- # Finding the exclude lines for else's is tricky, because they aren't
- # present in the compiler parse tree. Look at the previous suite,
- # and find its last line. If any line between there and the else's
- # first line are excluded, then we exclude the else.
- lastprev = self.getLastLine(prevsuite)
- firstelse = self.getFirstLine(suite)
- for l in range(lastprev+1, firstelse):
- if l in self.suite_spots:
- self.doSuite(None, suite, exclude=(l in self.excluded))
- break
- else:
- self.doSuite(None, suite)
-
- def doElse(self, prevsuite, node):
- if node.else_:
- self.doPlainWordSuite(prevsuite, node.else_)
-
- def visitFor(self, node):
- self.doSuite(node, node.body)
- self.doElse(node.body, node)
-
- visitWhile = visitFor
-
- def visitIf(self, node):
- # The first test has to be handled separately from the rest.
- # The first test is credited to the line with the "if", but the others
- # are credited to the line with the test for the elif.
- self.doSuite(node, node.tests[0][1])
- for t, n in node.tests[1:]:
- self.doSuite(t, n)
- self.doElse(node.tests[-1][1], node)
-
- def visitTryExcept(self, node):
- self.doSuite(node, node.body)
- for i in range(len(node.handlers)):
- a, b, h = node.handlers[i]
- if not a:
- # It's a plain "except:". Find the previous suite.
- if i > 0:
- prev = node.handlers[i-1][2]
- else:
- prev = node.body
- self.doPlainWordSuite(prev, h)
- else:
- self.doSuite(a, h)
- self.doElse(node.handlers[-1][2], node)
-
- def visitTryFinally(self, node):
- self.doSuite(node, node.body)
- self.doPlainWordSuite(node.body, node.final)
-
- def visitWith(self, node):
- self.doSuite(node, node.body)
-
- def visitGlobal(self, node):
- # "global" statements don't execute like others (they don't call the
- # trace function), so don't record their line numbers.
- pass
-
-the_coverage = None
-
-class CoverageException(Exception): pass
-
-class coverage:
- # Name of the cache file (unless environment variable is set).
- cache_default = ".coverage"
-
- # Environment variable naming the cache file.
- cache_env = "COVERAGE_FILE"
-
- # A dictionary with an entry for (Python source file name, line number
- # in that file) if that line has been executed.
- c = {}
-
- # A map from canonical Python source file name to a dictionary in
- # which there's an entry for each line number that has been
- # executed.
- cexecuted = {}
-
- # Cache of results of calling the analysis2() method, so that you can
- # specify both -r and -a without doing double work.
- analysis_cache = {}
-
- # Cache of results of calling the canonical_filename() method, to
- # avoid duplicating work.
- canonical_filename_cache = {}
-
- def __init__(self):
- global the_coverage
- if the_coverage:
- raise CoverageException, "Only one coverage object allowed."
- self.usecache = 1
- self.cache = None
- self.parallel_mode = False
- self.exclude_re = ''
- self.nesting = 0
- self.cstack = []
- self.xstack = []
- self.relative_dir = os.path.normcase(os.path.abspath(os.curdir)+os.sep)
- self.exclude('# *pragma[: ]*[nN][oO] *[cC][oO][vV][eE][rR]')
-
- # t(f, x, y). This method is passed to sys.settrace as a trace function.
- # See [van Rossum 2001-07-20b, 9.2] for an explanation of sys.settrace and
- # the arguments and return value of the trace function.
- # See [van Rossum 2001-07-20a, 3.2] for a description of frame and code
- # objects.
-
- def t(self, f, w, unused): #pragma: no cover
- if w == 'line':
- #print "Executing %s @ %d" % (f.f_code.co_filename, f.f_lineno)
- self.c[(f.f_code.co_filename, f.f_lineno)] = 1
- for c in self.cstack:
- c[(f.f_code.co_filename, f.f_lineno)] = 1
- return self.t
-
- def help(self, error=None): #pragma: no cover
- if error:
- print error
- print
- print __doc__
- sys.exit(1)
-
- def command_line(self, argv, help_fn=None):
- import getopt
- help_fn = help_fn or self.help
- settings = {}
- optmap = {
- '-a': 'annotate',
- '-c': 'collect',
- '-d:': 'directory=',
- '-e': 'erase',
- '-h': 'help',
- '-i': 'ignore-errors',
- '-m': 'show-missing',
- '-p': 'parallel-mode',
- '-r': 'report',
- '-x': 'execute',
- '-o:': 'omit=',
- }
- short_opts = string.join(map(lambda o: o[1:], optmap.keys()), '')
- long_opts = optmap.values()
- options, args = getopt.getopt(argv, short_opts, long_opts)
- for o, a in options:
- if o in optmap:
- settings[optmap[o]] = 1
- elif o + ':' in optmap:
- settings[optmap[o + ':']] = a
- elif o[2:] in long_opts:
- settings[o[2:]] = 1
- elif o[2:] + '=' in long_opts:
- settings[o[2:]+'='] = a
- else: #pragma: no cover
- pass # Can't get here, because getopt won't return anything unknown.
-
- if settings.get('help'):
- help_fn()
-
- for i in ['erase', 'execute']:
- for j in ['annotate', 'report', 'collect']:
- if settings.get(i) and settings.get(j):
- help_fn("You can't specify the '%s' and '%s' "
- "options at the same time." % (i, j))
-
- args_needed = (settings.get('execute')
- or settings.get('annotate')
- or settings.get('report'))
- action = (settings.get('erase')
- or settings.get('collect')
- or args_needed)
- if not action:
- help_fn("You must specify at least one of -e, -x, -c, -r, or -a.")
- if not args_needed and args:
- help_fn("Unexpected arguments: %s" % " ".join(args))
-
- self.parallel_mode = settings.get('parallel-mode')
- self.get_ready()
-
- if settings.get('erase'):
- self.erase()
- if settings.get('execute'):
- if not args:
- help_fn("Nothing to do.")
- sys.argv = args
- self.start()
- import __main__
- sys.path[0] = os.path.dirname(sys.argv[0])
- execfile(sys.argv[0], __main__.__dict__)
- if settings.get('collect'):
- self.collect()
- if not args:
- args = self.cexecuted.keys()
-
- ignore_errors = settings.get('ignore-errors')
- show_missing = settings.get('show-missing')
- directory = settings.get('directory=')
-
- omit = settings.get('omit=')
- if omit is not None:
- omit = omit.split(',')
- else:
- omit = []
-
- if settings.get('report'):
- self.report(args, show_missing, ignore_errors, omit_prefixes=omit)
- if settings.get('annotate'):
- self.annotate(args, directory, ignore_errors, omit_prefixes=omit)
-
- def use_cache(self, usecache, cache_file=None):
- self.usecache = usecache
- if cache_file and not self.cache:
- self.cache_default = cache_file
-
- def get_ready(self, parallel_mode=False):
- if self.usecache and not self.cache:
- self.cache = os.environ.get(self.cache_env, self.cache_default)
- if self.parallel_mode:
- self.cache += "." + gethostname() + "." + str(os.getpid())
- self.restore()
- self.analysis_cache = {}
-
- def start(self, parallel_mode=False):
- self.get_ready()
- if self.nesting == 0: #pragma: no cover
- sys.settrace(self.t)
- if hasattr(threading, 'settrace'):
- threading.settrace(self.t)
- self.nesting += 1
-
- def stop(self):
- self.nesting -= 1
- if self.nesting == 0: #pragma: no cover
- sys.settrace(None)
- if hasattr(threading, 'settrace'):
- threading.settrace(None)
-
- def erase(self):
- self.get_ready()
- self.c = {}
- self.analysis_cache = {}
- self.cexecuted = {}
- if self.cache and os.path.exists(self.cache):
- os.remove(self.cache)
-
- def exclude(self, re):
- if self.exclude_re:
- self.exclude_re += "|"
- self.exclude_re += "(" + re + ")"
-
- def begin_recursive(self):
- self.cstack.append(self.c)
- self.xstack.append(self.exclude_re)
-
- def end_recursive(self):
- self.c = self.cstack.pop()
- self.exclude_re = self.xstack.pop()
-
- # save(). Save coverage data to the coverage cache.
-
- def save(self):
- if self.usecache and self.cache:
- self.canonicalize_filenames()
- cache = open(self.cache, 'wb')
- import marshal
- marshal.dump(self.cexecuted, cache)
- cache.close()
-
- # restore(). Restore coverage data from the coverage cache (if it exists).
-
- def restore(self):
- self.c = {}
- self.cexecuted = {}
- assert self.usecache
- if os.path.exists(self.cache):
- self.cexecuted = self.restore_file(self.cache)
-
- def restore_file(self, file_name):
- try:
- cache = open(file_name, 'rb')
- import marshal
- cexecuted = marshal.load(cache)
- cache.close()
- if isinstance(cexecuted, types.DictType):
- return cexecuted
- else:
- return {}
- except:
- return {}
-
- # collect(). Collect data in multiple files produced by parallel mode
-
- def collect(self):
- cache_dir, local = os.path.split(self.cache)
- for f in os.listdir(cache_dir or '.'):
- if not f.startswith(local):
- continue
-
- full_path = os.path.join(cache_dir, f)
- cexecuted = self.restore_file(full_path)
- self.merge_data(cexecuted)
-
- def merge_data(self, new_data):
- for file_name, file_data in new_data.items():
- if file_name in self.cexecuted:
- self.merge_file_data(self.cexecuted[file_name], file_data)
- else:
- self.cexecuted[file_name] = file_data
-
- def merge_file_data(self, cache_data, new_data):
- for line_number in new_data.keys():
- if line_number not in cache_data:
- cache_data[line_number] = new_data[line_number]
-
- # canonical_filename(filename). Return a canonical filename for the
- # file (that is, an absolute path with no redundant components and
- # normalized case). See [GDR 2001-12-04b, 3.3].
-
- def canonical_filename(self, filename):
- if filename not in self.canonical_filename_cache:
- f = filename
- if os.path.isabs(f) and not os.path.exists(f):
- f = os.path.basename(f)
- if not os.path.isabs(f):
- for path in [os.curdir] + sys.path:
- g = os.path.join(path, f)
- if os.path.exists(g):
- f = g
- break
- cf = os.path.normcase(os.path.abspath(f))
- self.canonical_filename_cache[filename] = cf
- return self.canonical_filename_cache[filename]
-
- # canonicalize_filenames(). Copy results from "c" to "cexecuted",
- # canonicalizing filenames on the way. Clear the "c" map.
-
- def canonicalize_filenames(self):
- for filename, lineno in self.c.keys():
- if filename == '<string>':
- # Can't do anything useful with exec'd strings, so skip them.
- continue
- f = self.canonical_filename(filename)
- if f not in self.cexecuted:
- self.cexecuted[f] = {}
- self.cexecuted[f][lineno] = 1
- self.c = {}
-
- # morf_filename(morf). Return the filename for a module or file.
-
- def morf_filename(self, morf):
- if isinstance(morf, types.ModuleType):
- if not hasattr(morf, '__file__'):
- raise CoverageException, "Module has no __file__ attribute."
- f = morf.__file__
- else:
- f = morf
- return self.canonical_filename(f)
-
- # analyze_morf(morf). Analyze the module or filename passed as
- # the argument. If the source code can't be found, raise an error.
- # Otherwise, return a tuple of (1) the canonical filename of the
- # source code for the module, (2) a list of lines of statements
- # in the source code, (3) a list of lines of excluded statements,
- # and (4), a map of line numbers to multi-line line number ranges, for
- # statements that cross lines.
-
- def analyze_morf(self, morf):
- return self.analyze_morf(morf)[:-1]
-
- def analyze_morf2(self, morf):
- if morf in self.analysis_cache:
- return self.analysis_cache[morf]
- filename = self.morf_filename(morf)
- ext = os.path.splitext(filename)[1]
- if ext == '.pyc':
- if not os.path.exists(filename[0:-1]):
- raise CoverageException, ("No source for compiled code '%s'."
- % filename)
- filename = filename[0:-1]
- elif ext != '.py':
- raise CoverageException, "File '%s' not Python source." % filename
- source = open(filename, 'rU')
- lines, excluded_lines, line_map, definfo = \
- self.find_executable_statements2(source.read(),
- exclude=self.exclude_re)
- source.close()
- result = filename, lines, excluded_lines, line_map, definfo
- self.analysis_cache[morf] = result
- return result
-
- def first_line_of_tree(self, tree):
- while True:
- if len(tree) == 3 and type(tree[2]) == type(1):
- return tree[2]
- tree = tree[1]
-
- def last_line_of_tree(self, tree):
- while True:
- if len(tree) == 3 and type(tree[2]) == type(1):
- return tree[2]
- tree = tree[-1]
-
- def find_docstring_pass_pair(self, tree, spots):
- for i in range(1, len(tree)):
- if self.is_string_constant(tree[i]) and self.is_pass_stmt(tree[i+1]):
- first_line = self.first_line_of_tree(tree[i])
- last_line = self.last_line_of_tree(tree[i+1])
- self.record_multiline(spots, first_line, last_line)
-
- def is_string_constant(self, tree):
- try:
- return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.expr_stmt
- except:
- return False
-
- def is_pass_stmt(self, tree):
- try:
- return tree[0] == symbol.stmt and tree[1][1][1][0] == symbol.pass_stmt
- except:
- return False
-
- def record_multiline(self, spots, i, j):
- for l in range(i, j+1):
- spots[l] = (i, j)
-
- def get_suite_spots(self, tree, spots):
- """ Analyze a parse tree to find suite introducers which span a number
- of lines.
- """
- for i in range(1, len(tree)):
- if type(tree[i]) == type(()):
- if tree[i][0] == symbol.suite:
- # Found a suite, look back for the colon and keyword.
- lineno_colon = lineno_word = None
- for j in range(i-1, 0, -1):
- if tree[j][0] == token.COLON:
- # Colons are never executed themselves: we want the
- # line number of the last token before the colon.
- lineno_colon = self.last_line_of_tree(tree[j-1])
- elif tree[j][0] == token.NAME:
- if tree[j][1] == 'elif':
- # Find the line number of the first non-terminal
- # after the keyword.
- t = tree[j+1]
- while t and token.ISNONTERMINAL(t[0]):
- t = t[1]
- if t:
- lineno_word = t[2]
- else:
- lineno_word = tree[j][2]
- break
- elif tree[j][0] == symbol.except_clause:
- # "except" clauses look like:
- # ('except_clause', ('NAME', 'except', lineno), ...)
- if tree[j][1][0] == token.NAME:
- lineno_word = tree[j][1][2]
- break
- if lineno_colon and lineno_word:
- # Found colon and keyword, mark all the lines
- # between the two with the two line numbers.
- self.record_multiline(spots, lineno_word, lineno_colon)
-
- # "pass" statements are tricky: different versions of Python
- # treat them differently, especially in the common case of a
- # function with a doc string and a single pass statement.
- self.find_docstring_pass_pair(tree[i], spots)
-
- elif tree[i][0] == symbol.simple_stmt:
- first_line = self.first_line_of_tree(tree[i])
- last_line = self.last_line_of_tree(tree[i])
- if first_line != last_line:
- self.record_multiline(spots, first_line, last_line)
- self.get_suite_spots(tree[i], spots)
-
- def find_executable_statements(self, text, exclude=None):
- return self.find_executable_statements2(text, exclude)[:-1]
-
- def find_executable_statements2(self, text, exclude=None):
- # Find lines which match an exclusion pattern.
- excluded = {}
- suite_spots = {}
- if exclude:
- reExclude = re.compile(exclude)
- lines = text.split('\n')
- for i in range(len(lines)):
- if reExclude.search(lines[i]):
- excluded[i+1] = 1
-
- # Parse the code and analyze the parse tree to find out which statements
- # are multiline, and where suites begin and end.
- import parser
- tree = parser.suite(text+'\n\n').totuple(1)
- self.get_suite_spots(tree, suite_spots)
- #print "Suite spots:", suite_spots
-
- # Mapping from name of func/class -> (start, end).
- definfo = []
-
- # Use the compiler module to parse the text and find the executable
- # statements. We add newlines to be impervious to final partial lines.
- statements = {}
- ast = compiler.parse(text+'\n\n')
- visitor = StatementFindingAstVisitor(statements, excluded,
- suite_spots, definfo)
- compiler.walk(ast, visitor, walker=visitor)
-
- lines = statements.keys()
- lines.sort()
- excluded_lines = excluded.keys()
- excluded_lines.sort()
- return lines, excluded_lines, suite_spots, definfo
-
- # format_lines(statements, lines). Format a list of line numbers
- # for printing by coalescing groups of lines as long as the lines
- # represent consecutive statements. This will coalesce even if
- # there are gaps between statements, so if statements =
- # [1,2,3,4,5,10,11,12,13,14] and lines = [1,2,5,10,11,13,14] then
- # format_lines will return "1-2, 5-11, 13-14".
-
- def format_lines(self, statements, lines):
- pairs = []
- i = 0
- j = 0
- start = None
- pairs = []
- while i < len(statements) and j < len(lines):
- if statements[i] == lines[j]:
- if start is None:
- start = lines[j]
- end = lines[j]
- j = j + 1
- elif start:
- pairs.append((start, end))
- start = None
- i = i + 1
- if start:
- pairs.append((start, end))
- def stringify(pair):
- start, end = pair
- if start == end:
- return "%d" % start
- else:
- return "%d-%d" % (start, end)
- ret = string.join(map(stringify, pairs), ", ")
- return ret
-
- # Backward compatibility with version 1.
- def analysis(self, morf):
- f, s, _, m, mf = self.analysis2(morf)
- return f, s, m, mf
-
- def analysis2(self, morf):
- f, s, e, m, mf, _ = self.analysis3(morf)
- return f, s, e, m, mf
-
- def analysis3(self, morf):
- filename, statements, excluded, line_map, definfo = \
- self.analyze_morf2(morf)
- self.canonicalize_filenames()
- if filename not in self.cexecuted:
- self.cexecuted[filename] = {}
- missing = []
- for line in statements:
- lines = line_map.get(line, [line, line])
- for l in range(lines[0], lines[1]+1):
- if l in self.cexecuted[filename]:
- break
- else:
- missing.append(line)
- self.find_def_coverage(morf, statements, missing, definfo)
- return (filename, statements, excluded, missing,
- self.format_lines(statements, missing), definfo)
-
- def find_def_coverage(self, morf, statements, missing, definfo):
- """Return mapping from function name to coverage.
- """
- def_coverage = {}
- root = self.morf_name(morf)
- statements = set(statements)
- missing = set(missing)
- for info in definfo:
- if info.codestart is None:
- info.coverage = 1
- else:
- lines = set(range(info.codestart, info.end+1))
- stmt = len(lines.intersection(statements))
- miss = len(lines.intersection(missing))
- if miss == 0: info.cover = 1
- else: info.coverage = (1.0 - float(miss)/float(stmt))
-
- def relative_filename(self, filename):
- """ Convert filename to relative filename from self.relative_dir.
- """
- return filename.replace(self.relative_dir, "")
-
- def morf_name(self, morf):
- """ Return the name of morf as used in report.
- """
- if isinstance(morf, types.ModuleType):
- return morf.__name__
- else:
- return self.relative_filename(os.path.splitext(morf)[0])
-
- def filter_by_prefix(self, morfs, omit_prefixes):
- """ Return list of morfs where the morf name does not begin
- with any one of the omit_prefixes.
- """
- filtered_morfs = []
- for morf in morfs:
- for prefix in omit_prefixes:
- if self.morf_name(morf).startswith(prefix):
- break
- else:
- filtered_morfs.append(morf)
-
- return filtered_morfs
-
- def morf_name_compare(self, x, y):
- return cmp(self.morf_name(x), self.morf_name(y))
-
- def report(self, morfs, show_missing=1, ignore_errors=0, file=None, omit_prefixes=[]):
- if not isinstance(morfs, types.ListType):
- morfs = [morfs]
- # On windows, the shell doesn't expand wildcards. Do it here.
- globbed = []
- for morf in morfs:
- if isinstance(morf, strclass):
- globbed.extend(glob.glob(morf))
- else:
- globbed.append(morf)
- morfs = globbed
-
- morfs = self.filter_by_prefix(morfs, omit_prefixes)
- morfs.sort(self.morf_name_compare)
-
- max_name = max([5,] + map(len, map(self.morf_name, morfs)))
- fmt_name = "%%- %ds " % max_name
- fmt_err = fmt_name + "%s: %s"
- header = fmt_name % "Name" + " Stmts Exec Cover"
- fmt_coverage = fmt_name + "% 6d % 6d % 5d%%"
- if show_missing:
- header = header + " Missing"
- fmt_coverage = fmt_coverage + " %s"
- if not file:
- file = sys.stdout
- print >>file, header
- print >>file, "-" * len(header)
- total_statements = 0
- total_executed = 0
- for morf in morfs:
- name = self.morf_name(morf)
- try:
- _, statements, _, missing, readable = self.analysis2(morf)
- n = len(statements)
- m = n - len(missing)
- if n > 0:
- pc = 100.0 * m / n
- else:
- pc = 100.0
- args = (name, n, m, pc)
- if show_missing:
- args = args + (readable,)
- print >>file, fmt_coverage % args
- total_statements = total_statements + n
- total_executed = total_executed + m
- except KeyboardInterrupt: #pragma: no cover
- raise
- except:
- if not ignore_errors:
- typ, msg = sys.exc_info()[0:2]
- print >>file, fmt_err % (name, typ, msg)
- if len(morfs) > 1:
- print >>file, "-" * len(header)
- if total_statements > 0:
- pc = 100.0 * total_executed / total_statements
- else:
- pc = 100.0
- args = ("TOTAL", total_statements, total_executed, pc)
- if show_missing:
- args = args + ("",)
- print >>file, fmt_coverage % args
-
- # annotate(morfs, ignore_errors).
-
- blank_re = re.compile(r"\s*(#|$)")
- else_re = re.compile(r"\s*else\s*:\s*(#|$)")
-
- def annotate(self, morfs, directory=None, ignore_errors=0, omit_prefixes=[]):
- morfs = self.filter_by_prefix(morfs, omit_prefixes)
- for morf in morfs:
- try:
- filename, statements, excluded, missing, _ = self.analysis2(morf)
- self.annotate_file(filename, statements, excluded, missing, directory)
- except KeyboardInterrupt:
- raise
- except:
- if not ignore_errors:
- raise
-
- def annotate_file(self, filename, statements, excluded, missing, directory=None):
- source = open(filename, 'rU')
- if directory:
- dest_file = os.path.join(directory,
- os.path.basename(filename)
- + ',cover')
- else:
- dest_file = filename + ',cover'
- dest = open(dest_file, 'w')
- lineno = 0
- i = 0
- j = 0
- covered = 1
- while 1:
- line = source.readline()
- if line == '':
- break
- lineno = lineno + 1
- while i < len(statements) and statements[i] < lineno:
- i = i + 1
- while j < len(missing) and missing[j] < lineno:
- j = j + 1
- if i < len(statements) and statements[i] == lineno:
- covered = j >= len(missing) or missing[j] > lineno
- if self.blank_re.match(line):
- dest.write(' ')
- elif self.else_re.match(line):
- # Special logic for lines containing only 'else:'.
- # See [GDR 2001-12-04b, 3.2].
- if i >= len(statements) and j >= len(missing):
- dest.write('! ')
- elif i >= len(statements) or j >= len(missing):
- dest.write('> ')
- elif statements[i] == missing[j]:
- dest.write('! ')
- else:
- dest.write('> ')
- elif lineno in excluded:
- dest.write('- ')
- elif covered:
- dest.write('> ')
- else:
- dest.write('! ')
- dest.write(line)
- source.close()
- dest.close()
-
-# Singleton object.
-the_coverage = coverage()
-
-# Module functions call methods in the singleton object.
-def use_cache(*args, **kw):
- return the_coverage.use_cache(*args, **kw)
-
-def start(*args, **kw):
- return the_coverage.start(*args, **kw)
-
-def stop(*args, **kw):
- return the_coverage.stop(*args, **kw)
-
-def erase(*args, **kw):
- return the_coverage.erase(*args, **kw)
-
-def begin_recursive(*args, **kw):
- return the_coverage.begin_recursive(*args, **kw)
-
-def end_recursive(*args, **kw):
- return the_coverage.end_recursive(*args, **kw)
-
-def exclude(*args, **kw):
- return the_coverage.exclude(*args, **kw)
-
-def analysis(*args, **kw):
- return the_coverage.analysis(*args, **kw)
-
-def analysis2(*args, **kw):
- return the_coverage.analysis2(*args, **kw)
-
-def analysis3(*args, **kw):
- return the_coverage.analysis3(*args, **kw)
-
-def report(*args, **kw):
- return the_coverage.report(*args, **kw)
-
-def annotate(*args, **kw):
- return the_coverage.annotate(*args, **kw)
-
-def annotate_file(*args, **kw):
- return the_coverage.annotate_file(*args, **kw)
-
-# Save coverage data when Python exits. (The atexit module wasn't
-# introduced until Python 2.0, so use sys.exitfunc when it's not
-# available.)
-try:
- import atexit
- atexit.register(the_coverage.save)
-except ImportError:
- sys.exitfunc = the_coverage.save
-
-# Command-line interface.
-if __name__ == '__main__':
- the_coverage.command_line(sys.argv[1:])
-
-
-# A. REFERENCES
-#
-# [GDR 2001-12-04a] "Statement coverage for Python"; Gareth Rees;
-# Ravenbrook Limited; 2001-12-04;
-# <http://www.nedbatchelder.com/code/modules/rees-coverage.html>.
-#
-# [GDR 2001-12-04b] "Statement coverage for Python: design and
-# analysis"; Gareth Rees; Ravenbrook Limited; 2001-12-04;
-# <http://www.nedbatchelder.com/code/modules/rees-design.html>.
-#
-# [van Rossum 2001-07-20a] "Python Reference Manual (releae 2.1.1)";
-# Guide van Rossum; 2001-07-20;
-# <http://www.python.org/doc/2.1.1/ref/ref.html>.
-#
-# [van Rossum 2001-07-20b] "Python Library Reference"; Guido van Rossum;
-# 2001-07-20; <http://www.python.org/doc/2.1.1/lib/lib.html>.
-#
-#
-# B. DOCUMENT HISTORY
-#
-# 2001-12-04 GDR Created.
-#
-# 2001-12-06 GDR Added command-line interface and source code
-# annotation.
-#
-# 2001-12-09 GDR Moved design and interface to separate documents.
-#
-# 2001-12-10 GDR Open cache file as binary on Windows. Allow
-# simultaneous -e and -x, or -a and -r.
-#
-# 2001-12-12 GDR Added command-line help. Cache analysis so that it
-# only needs to be done once when you specify -a and -r.
-#
-# 2001-12-13 GDR Improved speed while recording. Portable between
-# Python 1.5.2 and 2.1.1.
-#
-# 2002-01-03 GDR Module-level functions work correctly.
-#
-# 2002-01-07 GDR Update sys.path when running a file with the -x option,
-# so that it matches the value the program would get if it were run on
-# its own.
-#
-# 2004-12-12 NMB Significant code changes.
-# - Finding executable statements has been rewritten so that docstrings and
-# other quirks of Python execution aren't mistakenly identified as missing
-# lines.
-# - Lines can be excluded from consideration, even entire suites of lines.
-# - The filesystem cache of covered lines can be disabled programmatically.
-# - Modernized the code.
-#
-# 2004-12-14 NMB Minor tweaks. Return 'analysis' to its original behavior
-# and add 'analysis2'. Add a global for 'annotate', and factor it, adding
-# 'annotate_file'.
-#
-# 2004-12-31 NMB Allow for keyword arguments in the module global functions.
-# Thanks, Allen.
-#
-# 2005-12-02 NMB Call threading.settrace so that all threads are measured.
-# Thanks Martin Fuzzey. Add a file argument to report so that reports can be
-# captured to a different destination.
-#
-# 2005-12-03 NMB coverage.py can now measure itself.
-#
-# 2005-12-04 NMB Adapted Greg Rogers' patch for using relative filenames,
-# and sorting and omitting files to report on.
-#
-# 2006-07-23 NMB Applied Joseph Tate's patch for function decorators.
-#
-# 2006-08-21 NMB Applied Sigve Tjora and Mark van der Wal's fixes for argument
-# handling.
-#
-# 2006-08-22 NMB Applied Geoff Bache's parallel mode patch.
-#
-# 2006-08-23 NMB Refactorings to improve testability. Fixes to command-line
-# logic for parallel mode and collect.
-#
-# 2006-08-25 NMB "#pragma: nocover" is excluded by default.
-#
-# 2006-09-10 NMB Properly ignore docstrings and other constant expressions that
-# appear in the middle of a function, a problem reported by Tim Leslie.
-# Minor changes to avoid lint warnings.
-#
-# 2006-09-17 NMB coverage.erase() shouldn't clobber the exclude regex.
-# Change how parallel mode is invoked, and fix erase() so that it erases the
-# cache when called programmatically.
-#
-# 2007-07-21 NMB In reports, ignore code executed from strings, since we can't
-# do anything useful with it anyway.
-# Better file handling on Linux, thanks Guillaume Chazarain.
-# Better shell support on Windows, thanks Noel O'Boyle.
-# Python 2.2 support maintained, thanks Catherine Proulx.
-#
-# 2007-07-22 NMB Python 2.5 now fully supported. The method of dealing with
-# multi-line statements is now less sensitive to the exact line that Python
-# reports during execution. Pass statements are handled specially so that their
-# disappearance during execution won't throw off the measurement.
-#
-# 2007-07-23 NMB Now Python 2.5 is *really* fully supported: the body of the
-# new with statement is counted as executable.
-#
-# 2007-07-29 NMB Better packaging.
-#
-# 2007-09-13 EDL Open Python files with 'rU' mode.
-#
-# 2007-09-20 EDL Added DefInfo, which gives fine-grained info about
-# coverage (and location) of functions & classes.
-
-# C. COPYRIGHT AND LICENCE
-#
-# Copyright 2001 Gareth Rees. All rights reserved.
-# Copyright 2004-2007 Ned Batchelder. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the
-# distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
-# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
-# DAMAGE.
-#
-# $Id: coverage.py 74 2007-07-29 22:28:35Z nedbat $
View
18 nltk/test/doctest_builder.py
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import
-import doctest
-from sphinx.ext.doctest import DocTestBuilder
-
-class NltkDocTestBuilder(DocTestBuilder):
- """
- Custom Sphinx doctest builder with NORMALIZE_WHITESPACE option added by default.
- """
-
- name = 'nltk-doctest'
-
- def init(self):
- super(NltkDocTestBuilder, self).init()
- self.opt |= doctest.NORMALIZE_WHITESPACE
-
-def setup(app):
- app.add_builder(NltkDocTestBuilder)
View
1,034 nltk/test/doctest_driver.py
@@ -1,1034 +0,0 @@
-#!/usr/bin/env python
-# A Test Driver for Doctest
-# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
-#
-# Provided as-is; use at your own risk; no warranty; no promises; enjoy!
-
-"""
-A driver for testing interactive python examples in text files and
-docstrings. This doctest driver performs three functions:
-
- - checking: Runs the interactive examples, and reports any examples
- whose actual output does not match their expected output.
-
- - debugging: Runs the interactive examples, and enters the debugger
- whenever an example's actual output does not match its expected
- output.
-
- - updating: Runs the interactive examples, and replaces the expected
- output with the actual output whenever they don't match. This is
- used to update the output for new or out-of-date examples.
-
-A number of other flags can be given; call the driver with the
-`--help` option for a complete list.
-"""
-
-import os, os.path, sys, unittest, pdb, bdb, re, tempfile, traceback
-import textwrap
-from doctest import *
-from doctest import DocTestCase, DocTestRunner
-from optparse import OptionParser, OptionGroup, Option
-from StringIO import StringIO
-import coverage
-
-# Use local NLTK.
-root_dir = os.path.abspath(os.path.join(sys.path[0], '..', '..'))
-sys.path.insert(0, root_dir)
-
-
-__version__ = '0.1'
-
-# Compiler flags: define '/' to do float division, even for ints.
-import __future__
-COMPILER_FLAGS = __future__.division.compiler_flag
-
-###########################################################################
-# Fix for unicode docstrings and Python 2*
-###########################################################################
-
-if __name__ == "__main__":
- import sys
- reload(sys)
- sys.setdefaultencoding("UTF-8")
- import doctest
- doctest.testmod()
-
-###########################################################################
-# Monkey-Patch to fix Doctest
-###########################################################################
-
-# The original version of this class has a bug that interferes with
-# code coverage functions. See: http://tinyurl.com/2htvfx
-class _OutputRedirectingPdb(pdb.Pdb):
- def __init__(self, out):
- self.__out = out
- self.__debugger_used = False
- pdb.Pdb.__init__(self)
-
- def set_trace(self):
- self.__debugger_used = True
- pdb.Pdb.set_trace(self)
-
- def set_continue(self):
- # Calling set_continue unconditionally would break unit test coverage
- # reporting, as Bdb.set_continue calls sys.settrace(None).
- if self.__debugger_used:
- pdb.Pdb.set_continue(self)
-
- def trace_dispatch(self, *args):
- save_stdout = sys.stdout
- sys.stdout = self.__out
- pdb.Pdb.trace_dispatch(self, *args)
- sys.stdout = save_stdout
-
-# Do the actual monkey-patching.
-import doctest
-doctest._OutputRedirectingPdb = _OutputRedirectingPdb
-
-###########################################################################
-# Utility Functions
-###########################################################################
-# These are copied from doctest; I don't import them because they're
-# private. See the versions in doctest for docstrings & comments.
-
-def _exception_traceback(exc_info):
- excout = StringIO()
- exc_type, exc_val, exc_tb = exc_info
- traceback.print_exception(exc_type, exc_val, exc_tb, file=excout)
- return excout.getvalue()
-
-class _SpoofOut(StringIO):
- def getvalue(self):
- result = StringIO.getvalue(self)
- if result and not result.endswith("\n"):
- result += "\n"
- if hasattr(self, "softspace"):
- del self.softspace
- return result
-
- def truncate(self, size=None):
- StringIO.truncate(self, size)
- if hasattr(self, "softspace"):
- del self.softspace
-
-###########################################################################
-# MyParser
-###########################################################################
-
-class MyDocTestParser(DocTestParser):
- PYLISTING_RE = re.compile(r'''
- (^\.\.[ ]*pylisting::.*\n # directive
- (?:[ ]*\n| # blank line or
- [ ]+.*\n)*) # indented line
- ''', re.VERBOSE+re.MULTILINE)
-
- # [xx] not used: split-pysrc_into_statements is used instead!
- PYLISTING_EX = re.compile(r'''
- (?:^[^ ].*\n # non-blank line
- (?:[ ]*\n | # blank line or
- [ ]+.*\n)*) # indented line
- ''', re.VERBOSE+re.MULTILINE)
-
- DOCTEST_OPTION_RE = re.compile(r'''
- ^[ ]*:\w+:.*\n # :option:
- (.*\S.*\n)* # non-blank lines
- ''', re.VERBOSE+re.MULTILINE)
-
- def parse(self, string, name='<string>'):
- output = []
- lineno_offset = 0
-
- for piecenum, piece in enumerate(self.PYLISTING_RE.split(string)):
- for example in DocTestParser.parse(self, piece, name):
- if isinstance(example, Example):
- example.lineno += lineno_offset
- output.append(example)
-
- # If we're inside a pylisting, then convert any
- # subpieces that are not marked by python prompts into
- # examples with an expected output of ''.
- elif piecenum%2 == 1 and example.strip():
- output.append(example[:example.find('\n')])
- # order matters here:
- pysrc = example[example.find('\n'):]
- pysrc = self.DOCTEST_OPTION_RE.sub('', pysrc)
- pysrc = textwrap.dedent(pysrc)
-
- #for ex in self.PYLISTING_EX.findall(pysrc):
- for ex in split_pysrc_into_statements(pysrc):
- source = ex.strip()
- if not source: continue
- want = ''
- exc_msg = None
- indent = 4 # close enough.
- lineno = lineno_offset # Not quite right!
- options = self._find_options(source, name, lineno)
- output.append(Example(source, want, exc_msg,
- lineno, indent, options))
- else:
- output.append(example)
-
- lineno_offset += piece.count('\n')
-
- # For debugging:
- #for ex in output:
- # if isinstance(ex, Example):
- # print '-'*70
- # print ex.source
- #output = []
-
- return output
-
- def get_examples(self, string, name='<string>'):
- examples = []
- ignore = False
-
- for x in self.parse(string, name):
- if isinstance(x, Example):
- if not ignore:
- examples.append(x)
- else:
- #print '.. doctest-ignore:: %s' % x.source.strip()[:50]
- pass
- else:
- if re.search(r'\.\.\s*doctest-ignore::?\s*$', x):
- ignore = True
- elif x.strip():
- ignore = False
- return examples
-
-###########################################################################
-# Update Runner
-###########################################################################
-
-class UpdateRunner(DocTestRunner):
- """
- A subclass of `DocTestRunner` that checks the output of each
- example, and replaces the expected output with the actual output
- for any examples that fail.
-
- `UpdateRunner` can be used:
- - To automatically fill in the expected output for new examples.
- - To correct examples whose output has become out-of-date.
-
- However, care must be taken not to update an example's expected
- output with an incorrect value.
- """
- def __init__(self, verbose=False, mark_updates=False):
- '''Construct a new update runner'''
- self._mark_updates = mark_updates
- DocTestRunner.__init__(self, verbose=verbose)
-
- def run(self, test, compileflags=None, out=None, clear_globs=True):
- '''Run the update runner'''
- self._new_want = {}
- (f,t) = DocTestRunner.run(self, test, compileflags, out, clear_globs)
-
- # Update the test's docstring, and the lineno's of the
- # examples, by breaking it into lines and replacing the old
- # expected outputs with the new expected outputs.
- old_lines = test.docstring.split('\n')
- new_lines = []
- lineno = 0
- offset = 0
-
- for example in test.examples:
- # Copy the lines up through the start of the example's
- # output from old_lines to new_lines.
- got_start = example.lineno + example.source.count('\n')
- new_lines += old_lines[lineno:got_start]
- lineno = got_start
- # Do a sanity check to make sure we're at the right lineno
- # (In particular, check that the example's expected output
- # appears in old_lines where we expect it to appear.)
- if example.want:
- assert (example.want.split('\n')[0] ==
- old_lines[lineno][example.indent:]), \
- 'Line number mismatch at %d' % lineno
- # Skip over the old expected output.
- old_len = example.want.count('\n')
- lineno += old_len
- # Mark any changes we make.
- if self._mark_updates and example in self._new_want:
- new_lines.append(' '*example.indent + '... ' +
- '# [!!] OUTPUT AUTOMATICALLY UPDATED [!!]')
- # Add the new expected output.
- new_want = self._new_want.get(example, example.want)
- if new_want:
- new_want = '\n'.join([' '*example.indent+l
- for l in new_want[:-1].split('\n')])
- new_lines.append(new_want)
- # Update the example's want & lieno fields
- example.want = new_want
- example.lineno += offset
- offset += example.want.count('\n') - old_len
- # Add any remaining lines
- new_lines += old_lines[lineno:]
-
- # Update the test's docstring.
- test.docstring = '\n'.join(new_lines)
-
- # Return failures & tries
- return (f,t)
-
- def report_start(self, out, test, example):
- pass
-
- def report_success(self, out, test, example, got):
- pass
-
- def report_unexpected_exception(self, out, test, example, exc_info):
- replacement = _exception_traceback(exc_info)
- self._new_want[example] = replacement
- if self._verbose:
- self._report_replacement(out, test, example, replacement)
-
- def report_failure(self, out, test, example, got):
- self._new_want[example] = got
- if self._verbose:
- self._report_replacement(out, test, example, got)
-
- def _report_replacement(self, out, test, example, replacement):
- want = '\n'.join([' '+l for l in example.want.split('\n')[:-1]])
- repl = '\n'.join([' '+l for l in replacement.split('\n')[:-1]])
- if want and repl:
- diff = 'Replacing:\n%s\nWith:\n%s\n' % (want, repl)
- elif want:
- diff = 'Removing:\n%s\n' % want
- elif repl:
- diff = 'Adding:\n%s\n' % repl
- out(self._header(test, example) + diff)
-
- DIVIDER = '-'*70
- def _header(self, test, example):
- if test.filename is None:
- tag = ("On line #%s of %s" %
- (example.lineno+1, test.name))
- elif test.lineno is None:
- tag = ("On line #%s of %s in %s" %
- (example.lineno+1, test.name, test.filename))
- else:
- lineno = test.lineno+example.lineno+1
- tag = ("On line #%s of %s (%s)" %
- (lineno, test.filename, test.name))
- source_lines = example.source.rstrip().split('\n')
- return (self.DIVIDER + '\n' + tag + '\n' +
- ' >>> %s\n' % source_lines[0] +
- ''.join([' ... %s\n' % l for l in source_lines[1:]]))
-
-###########################################################################
-# Debugger
-###########################################################################
-
-def _indent(s, indent=4):
- return re.sub('(?m)^(?!$)', indent*' ', s)
-
-import keyword, token, tokenize
-class Debugger:
- # Just using this for reporting:
- runner = DocTestRunner()
-
- def __init__(self, checker=None, set_trace=None):
- if checker is None:
- checker = OutputChecker()
- self.checker = checker
- if set_trace is None:
- set_trace = pdb.Pdb().set_trace
- self.set_trace = set_trace
-
- def _check_output(self, example):
- want = example.want
- optionflags = self._get_optionflags(example)
- got = sys.stdout.getvalue()
- sys.stdout.truncate(0)
- if not self.checker.check_output(want, got, optionflags):
- self.runner.report_failure(self.save_stdout.write,
- self.test, example, got)
- return False
- else:
- return True
-
- def _check_exception(self, example):
- want_exc_msg = example.exc_msg
- optionflags = self._get_optionflags(example)
- exc_info = sys.exc_info()
- got_exc_msg = traceback.format_exception_only(*exc_info[:2])[-1]
- if not self.checker.check_output(want_exc_msg, got_exc_msg,
- optionflags):
- got = _exception_traceback(exc_info)
- self.runner.report_failure(self.save_stdout.write,
- self.test, example, got)
- return False
- else:
- return True
-
- def _print_if_not_none(self, *args):
- if args == (None,):
- pass
- elif len(args) == 1:
- print `args[0]`
- else:
- print `args` # not quite right: >>> 1,
-
- def _comment_line(self, line):
- "Return a commented form of the given line"
- line = line.rstrip()
- if line:
- return '# '+line
- else:
- return '#'
-
- def _script_from_examples(self, s):
- output = []
- examplenum = 0
- for piece in MyDocTestParser().parse(s):
- if isinstance(piece, Example):
- self._script_from_example(piece, examplenum, output)
- examplenum += 1
- else:
- # Add non-example text.
- output += [self._comment_line(l)
- for l in piece.split('\n')[:-1]]
- # Combine the output, and return it.
- return '\n'.join(output)
-
- _CHK_OUT = 'if not CHECK_OUTPUT(__examples__[%d]): __set_trace__()'
- _CHK_EXC = 'if not CHECK_EXCEPTION(__examples__[%d]): __set_trace__()'
-
- def _script_from_example(self, example, i, output):
- source = self._simulate_compile_singlemode(example.source)[:-1]
-
- if example.exc_msg is None:
- output.append(source)
- output.append(self._CHK_OUT % i)
- else:
- output.append('try:')
- output.append(_indent(source))
- output.append(' '+self._CHK_OUT % i)
- output.append('except:')
- output.append(' '+self._CHK_EXC % i)
-
- def _simulate_compile_singlemode(self, s):
- # Calculate line offsets
- lines = [0, 0]
- pos = 0
- while 1:
- pos = s.find('\n', pos)+1
- if not pos: break
- lines.append(pos)
- lines.append(len(s))
-
- oldpos = 0
- parenlevel = 0
- deflevel = 0
- output = []
- stmt = []
-
- text = StringIO(s)
- tok_gen = tokenize.generate_tokens(text.readline)
- for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen:
- newpos = lines[srow] + scol
- stmt.append(s[oldpos:newpos])
- if tok != '':
- stmt.append(tok)
- oldpos = newpos + len(tok)
-
- # Update the paren level.
- if tok in '([{':
- parenlevel += 1
- if tok in '}])':
- parenlevel -= 1
-
- if tok in ('def', 'class') and deflevel == 0:
- deflevel = 1
- if deflevel and toktype == token.INDENT:
- deflevel += 1
- if deflevel and toktype == token.DEDENT:
- deflevel -= 1
-
- # Are we starting a statement?
- if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT,
- token.INDENT, token.ENDMARKER) or
- tok==':') and parenlevel == 0):
- if deflevel == 0 and self._is_expr(stmt[1:-2]):
- output += stmt[0]
- output.append('__print__((')
- output += stmt[1:-2]
- output.append('))')
- output += stmt[-2:]
- else:
- output += stmt
- stmt = []
- return ''.join(output)
-
- def _is_expr(self, stmt):
- stmt = [t for t in stmt if t]
- if not stmt:
- return False
-
- # An assignment signifies a non-exception, *unless* it
- # appears inside of parens (eg, ``f(x=1)``.)
- parenlevel = 0
- for tok in stmt:
- if tok in '([{': parenlevel += 1
- if tok in '}])': parenlevel -= 1
- if (parenlevel == 0 and
- tok in ('=', '+=', '-=', '*=', '/=', '%=', '&=', '+=',
- '^=', '<<=', '>>=', '**=', '//=')):
- return False
-
- # Any keywords *except* "not", "or", "and", "lambda", "in", "is"
- # signifies a non-expression.
- if stmt[0] in ("assert", "break", "class", "continue", "def",
- "del", "elif", "else", "except", "exec",
- "finally", "for", "from", "global", "if",
- "import", "pass", "print", "raise", "return",
- "try", "while", "yield"):
- return False
- return True
-
- def _get_optionflags(self, example):
- optionflags = 0
- for (flag, val) in example.options.items():
- if val:
- optionflags |= flag
- else:
- optionflags &= ~flag
- return optionflags
-
- def debug(self, test, pm=False):
- self.test = test
-
- # Save the old stdout
- self.save_stdout = sys.stdout
-
- # Convert the source docstring to a script.
- script = self._script_from_examples(test.docstring)
-
- # Create a debugger.
- debugger = _OutputRedirectingPdb(sys.stdout)
-
- # Patch pdb.set_trace to restore sys.stdout during interactive
- # debugging (so it's not still redirected to self._fakeout).
- save_set_trace = pdb.set_trace
- pdb.set_trace = debugger.set_trace
-
- # Write the script to a temporary file. Note that
- # tempfile.NameTemporaryFile() cannot be used. As the docs
- # say, a file so created cannot be opened by name a second
- # time on modern Windows boxes, and execfile() needs to open
- # it.
- srcfilename = tempfile.mktemp(".py", "doctestdebug_")
- f = open(srcfilename, 'w')
- f.write(script)
- f.close()
-
- # Set up the globals
- test.globs['CHECK_OUTPUT'] = self._check_output
- test.globs['CHECK_EXCEPTION'] = self._check_exception
- test.globs['__print__'] = self._print_if_not_none
- test.globs['__set_trace__'] = debugger.set_trace
- test.globs['__examples__'] = self.test.examples
- try:
- if pm is False:
- debugger.run("execfile(%r)" % srcfilename,
- test.globs, test.globs)
- else:
- try:
- sys.stdout = _SpoofOut()
- try:
- execfile(srcfilename, test.globs)
- except bdb.BdbQuit:
- return
- except:
- sys.stdout = self.save_stdout
- exc_info = sys.exc_info()
- exc_msg = traceback.format_exception_only(
- exc_info[0], exc_info[1])[-1]
- self.save_stdout.write(self.runner.DIVIDER+'\n')
- self.save_stdout.write('Unexpected exception:\n' +
- _indent(exc_msg))
- raise
- #self.post_mortem(debugger, exc_info[2])
- finally:
- sys.stdout = self.save_stdout
- finally:
- sys.set_trace = save_set_trace
- os.remove(srcfilename)
-
- def post_mortem(self, debugger, t):
- debugger.reset()
- while t.tb_next is not None:
- t = t.tb_next
- debugger.interaction(t.tb_frame, t)
-
-###########################################################################
-# Helper functions
-###########################################################################
-
-# Name can be:
-# - The filename of a text file
-# - The filename of a python file
-# - The dotted name of a python module
-
-# Return a list of test!
-def find(name):
- # Check for test names
- if ':' in name:
- (name, testname) = name.split(':')
- else:
- testname = None
-
- if os.path.exists(name):
- filename = os.path.normpath(os.path.abspath(name))
- ext = os.path.splitext(filename)[-1]
- if (ext[-3:] != '.py' and ext[-4:-1] != '.py'):
- # It's a text file; return the filename.
- if testname is not None:
- raise ValueError("test names can't be specified "
- "for text files")
- s = open(filename).read().decode('utf8')
- test = MyDocTestParser().get_doctest(s, {}, name, filename, 0)
- return [test]
- else:
- # It's a python file; import it. Make sure to set the
- # path correctly.
- basedir, modname = find_module_from_filename(filename)
- orig_path = sys.path[:]
- try:
- sys.path.insert(0, basedir)
- module = import_from_name(modname)
- finally:
- sys.path[:] = orig_path
- else:
- module = import_from_name(name)
-
- # Find tests.
- tests = DocTestFinder().find(module)
- if testname is not None:
- testname = '%s.%s' % (module.__name__, testname)
- tests = [t for t in tests if t.name.startswith(testname)]
- if len(tests) == 0:
- raise ValueError("test not found")
- return tests
-
-def import_from_name(name):
- try:
- return __import__(name, globals(), locals(), ['*'])
- except Exception, e:
- raise ValueError, str(e)
- except:
- raise ValueError, 'Error importing %r' % name
-
-def find_module_from_filename(filename):
- """
- Given a filename, return a tuple `(basedir, module)`, where
- `module` is the module's name, and `basedir` is the directory it
- should be loaded from (this directory should be added to the
- path to import it). Packages are handled correctly.
- """
- (basedir, file) = os.path.split(filename)
- (module_name, ext) = os.path.splitext(file)
-
- # If it's a package, then import with the directory name (don't
- # use __init__ as the module name).
- if module_name == '__init__':
- (basedir, module_name) = os.path.split(basedir)
-
- # If it's contained inside a package, then find the base dir.
- if (os.path.exists(os.path.join(basedir, '__init__.py')) or
- os.path.exists(os.path.join(basedir, '__init__.pyc')) or
- os.path.exists(os.path.join(basedir, '__init__.pyw'))):
- package = []
- while os.path.exists(os.path.join(basedir, '__init__.py')):
- (basedir,dir) = os.path.split(basedir)
- if dir == '': break
- package.append(dir)
- package.reverse()
- module_name = '.'.join(package+[module_name])
-
- return (basedir, module_name)
-
-def split_pysrc_into_statements(s):
- parens = 0 # Number of parens deep we're nested?
- quote = None # What type of string are we in (if any)?
- statements = [] # List of statements we've found
- continuation = False # Did last line end with a backslash?
- for line in s.lstrip().split('\n'):
- # Check indentation level.
- indent = re.match(r'\s*', line).end()
-
- # [DEBUG PRINTF]
- #print '%4d %6r %6s %5s %r' % (parens, quote, continuation,
- # indent, line[:40])
-
- # Add the line as a new statement or a continuation.
- if (parens == 0 and quote is None and indent == 0 and
- (not continuation)):
- if line.strip():
- statements.append(line)
- else:
- statements[-1] += '\n'+line
-
- # Scan the line, checking for quotes, parens, and comment
- # markers (so we can decide when a line is a continuation).
- line_has_comment = False
- for c in re.findall(r'\\.|"""|\'\'\'|"|\'|\(|\)|\[|\]|\{|\}|\#', line):
- if quote:
- if c == quote:
- quote = None
- elif c in '([{':
- parens += 1
- elif c in ')]}':
- parens -= 1
- elif c == '#':
- line_has_comment = True
- break
- elif c[0] != '\\':
- quote = c
- if not line_has_comment:
- continuation = line.strip().endswith('\\')
-
- return statements
-
-###########################################################################
-# Custom Checker, to ignore [# _foo] callouts in output.
-###########################################################################
-
-class MyOutputChecker(OutputChecker):
- CALLOUT_RE = re.compile(r' *#[ ]+\[_([\w-]+)\][ ]*$', re.MULTILINE)
- def check_output(self, want, got, optionflags):
- if OutputChecker.check_output(self, want, got, optionflags):
- return True
- else:
- want = self.CALLOUT_RE.sub('', want)
- return OutputChecker.check_output(self, want, got, optionflags)
-
-###########################################################################
-# Basic Actions
-###########################################################################
-
-# Finer control over output verbosity:
-from epydoc.cli import TerminalController
-
-class MyDocTestRunner(DocTestRunner):
- def __init__(self, checker=None, verbosity=1, optionflags=0,
- kbinterrupt_continue=False):
- DocTestRunner.__init__(self, checker, (verbosity>2), optionflags)
- self._verbosity = verbosity
- self._current_test = None
- self._term = TerminalController()
- self._stderr_term = TerminalController(sys.__stderr__)
- self._kbinterrupt_continue = kbinterrupt_continue
-
- def report_start(self, out, test, example):
- if 1 <= self._verbosity <= 2:
- src = example.source.split('\n')[0]
- if len(src) > 60: src = src[:57]+'...'
- if isinstance(src, unicode): src = src.encode('utf8')
- lineno = test.lineno + example.lineno + 1
- if self._verbosity == 1:
- if self._stderr_term.CLEAR_LINE:
- sys.__stderr__.write(self._stderr_term.CLEAR_LINE)
- else:
- sys.__stderr__.write('\n')
- sys.__stderr__.write('%s [Line %s] %s%s' %
- (self._stderr_term.BOLD, lineno,
- self._stderr_term.NORMAL, src))
- if self._verbosity == 2:
- sys.__stderr__.write('\n')
-
- else:
- DocTestRunner.report_start(self, out, test, example)
- sys.__stdout__.flush()
- self._current_test = (test, example)
-
- # Total hack warning: This munges the original source to
- # catch any keyboard interrupts, and turn them into special
- # ValueError interrupts.
- example.original_source = example.source
- if self._kbinterrupt_continue:
- example.source = ('try:\n%sexcept KeyboardInterrupt:\n '
- 'raise ValueError("KEYBOARD-INTERRUPT")\n' %
- doctest._indent(example.source))
-
- def report_failure(self, out, test, example, got):
- example.source = example.original_source
- if self._verbosity == 1:
- out('\n')
- out(self._failure_header(test, example) + self._term.RED+
- self._checker.output_difference(example, got, self.optionflags)+
- self._term.NORMAL)
-
- def report_unexpected_exception(self, out, test, example, exc_info):
- example.source = example.original_source
- if self._verbosity == 1:
- out('\n')
- out(self._failure_header(test, example) + self._term.RED)
- if (isinstance(exc_info[1], ValueError) and
- exc_info[1].args[0] == 'KEYBOARD-INTERRUPT'):
- out(self._term.RED+self._term.BOLD)
- out('Keyboard interrupt; Continuing!\n\n' + self._term.NORMAL)
- else:
- out('Exception raised:\n' + self._term.NORMAL +
- _indent(_exception_traceback(exc_info)))
-
- def _failure_header(self, test, example):
- out = (self._term.CYAN+self._term.BOLD+'*'*75+self._term.NORMAL+'\n')
- out += (self._term.GREEN)
- if test.filename:
- if test.lineno is not None and example.lineno is not None:
- lineno = test.lineno + example.lineno + 1
- else:
- lineno = '?'
- out += ('File "%s", line %s, in %s\n' %
- (test.filename, lineno, test.name))
- else:
- out += ('Line %s, in %s\n' % (example.lineno+1, test.name))
- out += (self._term.RED)
- out += ('Failed example:\n')
- source = example.source
- out += (_indent(source))
- if isinstance(out, unicode): out = out.encode('utf8')
- return out
-
- def run(self, test, compileflags=None, out=None, clear_globs=True):
- save_stderr = sys.stderr
- sys.stderr = _SpoofOut()
-
- if self._verbosity > 0:
- print >>save_stderr, (
- self._stderr_term.CYAN+self._stderr_term.BOLD+
- 'Testing %s...'%test.name+self._stderr_term.NORMAL)
- try:
- fails, tries = DocTestRunner.run(self, test, compileflags,
- out, clear_globs)
- except KeyboardInterrupt:
- if self._current_test is None: raise
-
- print >>save_stderr, self._failure_header(*self._current_test)
- print >>save_stderr, (
- self._stderr_term.RED+self._stderr_term.BOLD+
- 'Keyboard Interrupt!'+self._stderr_term.NORMAL)
- if self._verbosity == 1:
- save_stderr.write(self._stderr_term.CLEAR_LINE)
- if self._verbosity > 0:
- if fails:
- print >>save_stderr, (
- self._stderr_term.RED+self._stderr_term.BOLD+
- ' %d example(s) failed!'%fails+self._stderr_term.NORMAL)
- else:
- print >>save_stderr, (
- self._stderr_term.GREEN+self._stderr_term.BOLD+
- ' All examples passed'+self._stderr_term.NORMAL)
- print >>save_stderr
- sys.stderr = save_stderr
-
-def run(names, optionflags, verbosity, kbinterrupt_continue):
- checker = MyOutputChecker()
- runner = MyDocTestRunner(checker=checker, verbosity=verbosity,
- optionflags=optionflags,
- kbinterrupt_continue=kbinterrupt_continue)
- for name in names:
- try: tests = find(name)
- except ValueError, e:
- print >>sys.stderr, ('%s: Error processing %s -- %s' %
- (sys.argv[0], name, e))
- continue
- for test in tests:
- runner.run(test, COMPILER_FLAGS)
- if verbosity == 1:
- sys.stdout.write('.')
- sys.stdout.flush(); sys.stderr.flush()
- return runner
-
- # temporary hack:
-# for name in names:
-# testfile(name, optionflags=optionflags, verbose=True,
-# module_relative=False)
-# return
-
- suite = unittest.TestSuite()
- for name in names:
- try:
- for test in find(name):
- suite.addTest(DocTestCase(test, optionflags))
- except ValueError, e:
- print >>sys.stderr, ('%s: Error processing %s -- %s' %
- (sys.argv[0], name, e))
- unittest.TextTestRunner(verbosity=verbosity).run(suite)
-
-def debug(names, optionflags, verbosity, pm=True):
- debugger = Debugger()
- for name in names:
- try:
- for test in find(name):
- debugger.debug(test, pm)
- except ValueError, e:
- raise
- print >>sys.stderr, ('%s: Error processing %s -- %s' %
- (sys.argv[0], name, e))
-
-def update(names, optionflags, verbosity):
- runner = UpdateRunner(verbose=True)
- for name in names:
- try:
- # Make sure we're running on a text file.
- tests = find(name)
- if len(tests) != 1 or tests[0].lineno != 0:
- raise ValueError('update can only be used with text files')
- test = tests[0]
-
- # Run the updater!
- (failures, tries) = runner.run(test)
-
- # Confirm the changes.
- if failures == 0:
- print 'No updates needed!'
- else:
- print '*'*70
- print '%d examples updated.' % failures
- print '-'*70
- sys.stdout.write('Accept updates? [y/N] ')
- sys.stdout.flush()
- if sys.stdin.readline().lower().strip() in ('y', 'yes'):
- # Make a backup of the original contents.
- backup = test.filename+'.bak'
- print 'Renaming %s -> %s' % (name, backup)
- os.rename(test.filename, backup)
- # Write the new contents.
- print 'Writing updated version to %s' % test.filename
- out = open(test.filename, 'w')
- out.write(test.docstring)
- out.close()
- else:
- print 'Updates rejected!'
- except ValueError, e:
- raise
- print >>sys.stderr, ('%s: Error processing %s -- %s' %
- (sys.argv[0], name, e))
-
-###########################################################################
-# Main script
-###########################################################################
-
-# Action options
-CHECK_OPT = Option("--check",
- action="store_const", dest="action", const="check",
- default="check",
- help="Verify the output of the doctest examples in the "
- "given files.")
-
-UPDATE_OPT = Option("--update", "-u",
- action="store_const", dest="action", const="update",
- help="Update the expected output for new or out-of-date "
- "doctest examples in the given files. In "
- "particular, find every example whose actual output "
- "does not match its expected output; and replace its "
- "expected output with its actual output. You will "
- "be asked to verify the changes before they are "
- "written back to the file; be sure to check them over "
- "carefully, to ensure that you don't accidentally "
- "create broken test cases.")
-
-DEBUG_OPT = Option("--debug",
- action="store_const", dest="action", const="debug",
- help="Verify the output of the doctest examples in the "
- "given files. If any example fails, then enter the "
- "python debugger.")
-
-# Reporting options
-VERBOSE_OPT = Option("-v", "--verbose",
- action="count", dest="verbosity", default=1,
- help="Increase verbosity.")
-
-QUIET_OPT = Option("-q", "--quiet",
- action="store_const", dest="verbosity", const=0,
- help="Decrease verbosity.")
-
-UDIFF_OPT = Option("--udiff", '-d',
- action="store_const", dest="udiff", const=1, default=0,
- help="Display test failures using unified diffs.")
-
-CDIFF_OPT = Option("--cdiff",
- action="store_const", dest="cdiff", const=1, default=0,
- help="Display test failures using context diffs.")
-
-NDIFF_OPT = Option("--ndiff",
- action="store_const", dest="ndiff", const=1, default=0,
- help="Display test failures using ndiffs.")
-
-COVERAGE_OPT = Option("--coverage",
- action="store", dest="coverage", metavar='FILENAME',
- help="Generate coverage information, and write it to the "
- "given file.")
-
-CONTINUE_OPT = Option("--continue", dest='kbinterrupt_continue',
- action='store_const', const=1, default=0,
- help="If a test is interrupted by a keyboard "
- "interrupt, then report the interrupt and continue")
-
-# Output Comparison options
-IGNORE_EXCEPTION_DETAIL_OPT = Option("--ignore_exception_detail",
- action="store_const", dest="ignore_exception_detail", const=1, default=0,
- help="Ignore exception details in the expected output.")
-ELLIPSIS_OPT = Option("--ellipsis",
- action="store_const", dest="ellipsis", const=1, default=0,
- help="Allow \"...\" to be used for ellipsis in the "
- "expected output.")
-NORMWS_OPT = Option("--normalize_whitespace",
- action="store_const", dest="normws", const=1, default=1,
- help="Ignore whitespace differences between "
- "the expected output and the actual output.")
-
-def main():
- # Create the option parser.
- optparser = OptionParser(usage='%prog [options] NAME ...',
- version="Edloper's Doctest Driver, "
- "version %s" % __version__)
-
- action_group = OptionGroup(optparser, 'Actions (default=check)')
- action_group.add_options([CHECK_OPT, UPDATE_OPT, DEBUG_OPT])
- optparser.add_option_group(action_group)
-
- reporting_group = OptionGroup(optparser, 'Reporting')
- reporting_group.add_options([VERBOSE_OPT, QUIET_OPT,
- UDIFF_OPT, CDIFF_OPT, NDIFF_OPT,
- COVERAGE_OPT, CONTINUE_OPT])
- optparser.add_option_group(reporting_group)
-
- compare_group = OptionGroup(optparser, 'Output Comparison')
- compare_group.add_options([IGNORE_EXCEPTION_DETAIL_OPT, ELLIPSIS_OPT, NORMWS_OPT])
- optparser.add_option_group(compare_group)
-
- # Extract optionflags and the list of file names.
- optionvals, names = optparser.parse_args()
- if len(names) == 0:
- optparser.error("No files specified")
- optionflags = (optionvals.udiff * REPORT_UDIFF |
- optionvals.cdiff * REPORT_CDIFF |
- optionvals.ellipsis * ELLIPSIS |
- optionvals.ignore_exception_detail * IGNORE_EXCEPTION_DETAIL |
- optionvals.normws * NORMALIZE_WHITESPACE)
-
- # Check coverage, if requested
- if optionvals.coverage:
- coverage.use_cache(True, cache_file=optionvals.coverage)
- coverage.start()
-
- # Perform the requested action.
- if optionvals.action == 'check':
- run(names, optionflags, optionvals.verbosity,
- optionvals.kbinterrupt_continue)
- elif optionvals.action == 'update':
- update(names, optionflags, optionvals.verbosity)
- elif optionvals.action == 'debug':
- debug(names, optionflags, optionvals.verbosity)
- else:
- optparser.error('INTERNAL ERROR: Bad action %s' % optionvals.action)
-
- # Check coverage, if requested
- if optionvals.coverage:
- coverage.stop()
-
-if __name__ == '__main__': main()
View
131 nltk/test/doctest_nose_plugin.py
@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""
+Patched version of nose doctest plugin.
+See https://github.com/nose-devs/nose/issues/7
+"""
+from nose.plugins.doctests import *
+
+class DoctestFix(Doctest):
+
+ def options(self, parser, env):
+ super(DoctestFix, self).options(parser, env)
+ parser.add_option('--doctest-options', action="append",
+ dest="doctestOptions",
+ metavar="OPTIONS",
+ help="Specify options to pass to doctest. " +
+ "Eg. '+ELLIPSIS,+NORMALIZE_WHITESPACE'")
+
+ def configure(self, options, config):
+ super(DoctestFix, self).configure(options, config)
+ self.optionflags = 0
+ if options.doctestOptions:
+ flags = ",".join(options.doctestOptions).split(',')
+ for flag in flags:
+ try:
+ if flag.startswith('+'):
+ self.optionflags |= getattr(doctest, flag[1:])
+ elif flag.startswith('-'):
+ self.optionflags &= ~getattr(doctest, flag[1:])
+ else:
+ raise ValueError(
+ "Must specify doctest options with starting " +