Cheetah patches:

Add 'untaint' hook.
Enable #filter to use python scope variables
    (rather than magic cheetah lookup)
Some pyflakes cleanup.

All of this needs unit testing still.

cheetah/Compiler.py

@@ -11,25 +11,26 @@
 import sys
 import os
 import os.path
-from os.path import getmtime, exists
+from os.path import getmtime
 import re
-import types
 import time
 import random
 import warnings
 import copy
 import codecs
 
+#TODO(buck|2011-11-22): remove all this random crap. I'm pretty certain it's unecessary.
+random.seed(0) # random, but deterministic
+
 from Cheetah.Version import Version, VersionTuple
 from Cheetah.SettingsManager import SettingsManager
 from Cheetah.Utils.Indenter import indentize # an undocumented preprocessor
-from Cheetah import ErrorCatchers
 from Cheetah import NameMapper
 from Cheetah.Parser import Parser, ParseError, specialVarRE, \
-     STATIC_CACHE, REFRESH_CACHE, SET_LOCAL, SET_GLOBAL, SET_MODULE, \
+     STATIC_CACHE, REFRESH_CACHE, SET_GLOBAL, SET_MODULE, \
      unicodeDirectiveRE, encodingDirectiveRE, escapedNewlineRE
 
-from Cheetah.NameMapper import NotFound, valueForName, valueFromSearchList, valueFromFrameOrSearchList
+from Cheetah.NameMapper import valueForName, valueFromSearchList, valueFromFrameOrSearchList
 VFFSL=valueFromFrameOrSearchList
 VFSL=valueFromSearchList
 VFN=valueForName
@@ -459,7 +460,11 @@ def commitStrConst(self):
             out.append('"""')
             out.append(body)
             out.append('"""')
-        self.addWriteChunk(''.join(out))
+
+        if self.setting('useFilters'):
+            self.addWriteChunk('_untaint(' + ''.join(out) + ')')
+        else:
+            self.addWriteChunk(''.join(out))
 
     def handleWSBeforeDirective(self):
         """Truncate the pending strCont to the beginning of the current line.
@@ -932,7 +937,7 @@ def setTransform(self, transformer, isKlass):
         self.addChunk('trans._response._filter = %s' % transformer)
         self.addChunk('write = trans._response.write')
 
-    def setFilter(self, theFilter, isKlass):
+    def setFilter(self, theFilter, untaint, isKlass):
         class FilterDetails:
             pass
         filterDetails = FilterDetails()
@@ -955,18 +960,30 @@ class FilterDetails:
                 self.indent()
                 self.addChunk('_filter = self._CHEETAH__currentFilter = self._CHEETAH__filters[filterName]')
                 self.dedent()
-                self.addChunk('else:')
+                self.addChunk('elif hasattr(self._CHEETAH__filtersLib, filterName):')
                 self.indent()
                 self.addChunk('_filter = self._CHEETAH__currentFilter'
-                              +' = \\\n\t\t\tself._CHEETAH__filters[filterName] = '
+                              + ' = \\\n\t\t\tself._CHEETAH__filters[filterName] = '
                               + 'getattr(self._CHEETAH__filtersLib, filterName)(self).filter')
                 self.dedent()
+                self.addChunk('else:')
+                self.indent()
+                self.addChunk('_filter = self._CHEETAH__currentFilter'
+                              + ' = \\\n\t\t\tself._CHEETAH__filters[filterName] = '
+                              + theFilter)
+                self.dedent()
+
+		# TODO: unit test the untaint=None case
+		self.addChunk('_orig_untaint = _untaint')
+		self.addChunk('_untaint = self._CHEETAH__currentUntaint = ' + untaint)
 
+
     def closeFilterBlock(self):
         ID, filterDetails = self._filterRegionsStack.pop()
         #self.addChunk('_filter = self._CHEETAH__initialFilter')
         #self.addChunk('_filter = _orig_filter%(ID)s'%locals())
         self.addChunk('_filter = self._CHEETAH__currentFilter = _orig_filter%(ID)s'%locals())
+        self.addChunk('_untaint = self._CHEETAH__currentUntaint = _orig_untaint')
 
 class AutoMethodCompiler(MethodCompiler):
 
@@ -1023,6 +1040,10 @@ def cleanupState(self):
                 else:
                     self._streamingEnabled = False
 
+        if not self._methodBodyChunks:
+            # Some filters only work if there's always at least one chunk.
+            self.addFilteredChunk("''")
+
         self._indentLev = self.setting('initialMethIndentLevel')
         mainBodyChunks = self._methodBodyChunks
         self._methodBodyChunks = []
@@ -1071,6 +1092,7 @@ def _addAutoSetupCode(self):
                 self.addChunk('_filter = lambda x, **kwargs: unicode(x)')
             else:
                 self.addChunk('_filter = self._CHEETAH__currentFilter')
+            self.addChunk('_untaint = self._CHEETAH__untaint')
         self.addChunk('')
         self.addChunk("#" *40)
         self.addChunk('## START - generated method body')
@@ -1783,9 +1805,6 @@ def setCompilerSetting(self, key, valueExpr):
 
     def setCompilerSettings(self, keywords, settingsStr):
         KWs = keywords
-        merge = True
-        if 'nomerge' in KWs:
-            merge = False
 
         if 'reset' in KWs:
             # @@TR: this is actually caught by the parser at the moment.

cheetah/DummyTransaction.py

@@ -54,16 +54,28 @@ def writeln(self, txt):
     def getvalue(self, outputChunks=None):
         chunks = outputChunks or self._outputChunks
         try:
-            return u''.join(chunks)
+            return concat(chunks)
         except UnicodeDecodeError, ex:
             logging.debug('Trying to work around a UnicodeDecodeError in getvalue()')
             logging.debug('...perhaps you could fix "%s" while you\'re debugging')
-            return ''.join((self.safeConvert(c) for c in chunks))
+            return concat(self.safeConvert(c) for c in chunks)
 
     def writelines(self, *lines):
         ## not used
         [self.writeln(ln) for ln in lines]
 
+def concat(strings):
+        strings = iter(strings)
+
+        try:
+                result = strings.next()
+        except StopIteration:
+                return u''
+
+        for string in strings:
+                result += string
+        return result
+
 
 class DummyTransaction(object):
     '''

cheetah/Parser.py

@@ -2493,13 +2493,27 @@ def eatFilter(self):
         else:
             isKlass = False
             theFilter = self.getIdentifier()
+        self.getWhiteSpace()
+
+        #TODO(buck|2011-11-30): unit test this extended statement
+        # "#filter a b" should throw an error.
+        if not self.atEnd() and self.peek()==',':
+            self.advance() # skip over :
             self.getWhiteSpace()
+            if self.matchCheetahVarStart():
+                untaint = self.getExpression(pyTokensToBreakAt=[':'])
+            else:
+                untaint = self.getCheetahVar(plain=True, skipStartToken=True)
+        else:
+            untaint = None
+        self.getWhiteSpace()
+
         theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos)
 
         if self.matchColonForSingleLineShortFormDirective():
             self.advance() # skip over :
             self.getWhiteSpace(max=1)
-            self._compiler.setFilter(theFilter, isKlass)
+            self._compiler.setFilter(theFilter, untaint, isKlass)
             self.parse(breakPoint=self.findEOL(gobble=False))
             self._compiler.closeFilterBlock()
         else:
@@ -2508,7 +2522,7 @@ def eatFilter(self):
             self.getWhiteSpace()
             self.pushToOpenDirectivesStack("filter")
             self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
-            self._compiler.setFilter(theFilter, isKlass)
+            self._compiler.setFilter(theFilter, untaint, isKlass)
 
     def eatTransform(self):
         isLineClearToStartToken = self.isLineClearToStartToken()

cheetah/Template.py

@@ -1050,6 +1050,7 @@ def __init__(self, source=None,
 
                  file=None,
                  filter='RawOrEncodedUnicode', # which filter from Cheetah.Filters
+                 untaint=None,
                  filtersLib=Filters,
                  errorCatcher=None,
 
@@ -1247,7 +1248,7 @@ def __init__(self, source=None,
 
         self._initCheetahInstance(
             searchList=searchList, namespaces=namespaces,
-            filter=filter, filtersLib=filtersLib,
+            filter=filter, untaint=untaint, filtersLib=filtersLib,
             errorCatcher=errorCatcher,
             _globalSetVars=_globalSetVars,
             compilerSettings=compilerSettings,
@@ -1445,6 +1446,7 @@ def _initCheetahInstance(self,
                              searchList=None,
                              namespaces=None,
                              filter='RawOrEncodedUnicode', # which filter from Cheetah.Filters
+                             untaint=None,
                              filtersLib=Filters,
                              errorCatcher=None,
                              _globalSetVars=None,
@@ -1503,6 +1505,10 @@ def _initCheetahInstance(self,
         self._CHEETAH__currentFilter = self._CHEETAH__filters[filterName] = klass(self).filter
         self._CHEETAH__initialFilter = self._CHEETAH__currentFilter
 
+        if untaint is None:
+            untaint = lambda x: x
+        self._CHEETAH__untaint = untaint
+
         self._CHEETAH__errorCatchers = {}
         if errorCatcher:
             if isinstance(errorCatcher, basestring):