merged toby-stiff-cache-branch and toby-unicode-branch

DT_Util.py

@@ -10,11 +10,11 @@
 # FOR A PARTICULAR PURPOSE
 # 
 ##############################################################################
-'''$Id: DT_Util.py,v 1.85 2001/11/28 15:50:55 matt Exp $''' 
-__version__='$Revision: 1.85 $'[11:-2]
+'''$Id: DT_Util.py,v 1.86 2002/03/27 10:14:02 htrd Exp $''' 
+__version__='$Revision: 1.86 $'[11:-2]
 
 import re, os
-from html_quote import html_quote # for import by other modules, dont remove!
+from html_quote import html_quote, ustr # for import by other modules, dont remove!
 from RestrictedPython.Guards import safe_builtins
 from RestrictedPython.Utilities import utility_builtins
 from RestrictedPython.Eval import RestrictionCapableEval
@@ -41,10 +41,9 @@ def int_param(params,md,name,default=0, st=type('')):
 try:
     import ExtensionClass
     from cDocumentTemplate import InstanceDict, TemplateDict, \
-         render_blocks, safe_callable
+         render_blocks, safe_callable, join_unicode
 except: from pDocumentTemplate import InstanceDict, TemplateDict, \
-        render_blocks, safe_callable
-
+        render_blocks, safe_callable, join_unicode
 
 functype = type(int_param)
 class NotBindable:

DT_Var.py

@@ -145,10 +145,10 @@
    
 
 ''' # '
-__rcs_id__='$Id: DT_Var.py,v 1.51 2002/03/18 20:15:27 andreasjung Exp $'
-__version__='$Revision: 1.51 $'[11:-2]
+__rcs_id__='$Id: DT_Var.py,v 1.52 2002/03/27 10:14:02 htrd Exp $'
+__version__='$Revision: 1.52 $'[11:-2]
 
-from DT_Util import parse_params, name_param, str
+from DT_Util import parse_params, name_param, str, ustr
 import os, string, re,  sys
 from urllib import quote, quote_plus
 from cgi import escape
@@ -253,7 +253,7 @@ def render(self, md):
 
         # finally, pump it through the actual string format...
         fmt=self.fmt
-        if fmt=='s': val=str(val)
+        if fmt=='s': val=ustr(val)
         else: val = ('%'+self.fmt) % (val,)
 
         # next, look for upper, lower, etc

cDocumentTemplate.c

@@ -12,7 +12,7 @@
  ****************************************************************************/
 static char cDocumentTemplate_module_documentation[] = 
 ""
-"\n$Id: cDocumentTemplate.c,v 1.43 2002/03/21 15:48:54 htrd Exp $"
+"\n$Id: cDocumentTemplate.c,v 1.44 2002/03/27 10:14:02 htrd Exp $"
 ;
 
 #include "ExtensionClass.h"
@@ -22,7 +22,7 @@ static PyObject *py___call__, *py___roles__, *py_AUTHENTICATED_USER;
 static PyObject *py_hasRole, *py__proxy_roles, *py_Unauthorized;
 static PyObject *py_Unauthorized_fmt, *py_guarded_getattr;
 static PyObject *py__push, *py__pop, *py_aq_base, *py_renderNS;
-static PyObject *py___class__, *html_quote;
+static PyObject *py___class__, *html_quote, *ustr;
 
 /* ----------------------------------------------------- */
 
@@ -42,6 +42,8 @@ typedef struct {
 
 staticforward PyExtensionClass InstanceDictType;
 
+staticforward PyObject *_join_unicode(PyObject *prejoin);
+
 static PyObject *
 InstanceDict___init__(InstanceDictobject *self, PyObject *args)
 {
@@ -685,22 +687,43 @@ render_blocks_(PyObject *blocks, PyObject *rendered,
 	      if (PyString_Check(t)) t=PyObject_GetItem(md, t);
 	      else t=PyObject_CallObject(t, mda);
 
-              if (t == NULL || (! PyString_Check(t)))
+              if (t == NULL) return -1;
+
+              if (! ( PyString_Check(t) || PyUnicode_Check(t) ) )
                 {
-                  if (t) ASSIGN(t, PyObject_Str(t));
+                  ASSIGN(t, PyObject_CallFunction(ustr, "O", t));
                   UNLESS(t) return -1;
                 }
 
-              if (PyString_Check(t) 
-                  && PyTuple_GET_SIZE(block) == 3) /* html_quote */
+              if (PyTuple_GET_SIZE(block) == 3) /* html_quote */
                 {
-                  if (strchr(PyString_AS_STRING(t), '&')
-                      || strchr(PyString_AS_STRING(t), '<')
-                      || strchr(PyString_AS_STRING(t), '>')
-                      || strchr(PyString_AS_STRING(t), '"')
-                      )
-                    ASSIGN(t, PyObject_CallFunction(html_quote, "O", t));
-		    if (t == NULL) return -1;
+                  int skip_html_quote;
+                  if (PyString_Check(t))
+                    {
+                      if (strchr(PyString_AS_STRING(t), '&') ||
+                          strchr(PyString_AS_STRING(t), '<') ||
+                          strchr(PyString_AS_STRING(t), '>') ||
+                          strchr(PyString_AS_STRING(t), '"')     )
+                        {
+                          /* string includes html problem characters, so
+                             we cant skip the quoting process */
+                          skip_html_quote = 0;
+                        }
+                      else
+                        {
+                          skip_html_quote = 1;
+                        }
+                    }
+                  else
+                    {
+                      /* never skip the quoting for unicode strings */
+                      skip_html_quote = 0;
+                    }
+                  if (!skip_html_quote)
+                    {
+                      ASSIGN(t, PyObject_CallFunction(html_quote, "O", t));
+                      if (t == NULL) return -1;
+                    }
                 }
 
               block = t;
@@ -787,7 +810,7 @@ render_blocks_(PyObject *blocks, PyObject *rendered,
               return -1;
             }
         }
-      else if (PyString_Check(block))
+      else if (PyString_Check(block) || PyUnicode_Check(block))
 	{
 	  Py_INCREF(block);
 	}
@@ -830,7 +853,7 @@ render_blocks(PyObject *self, PyObject *args)
   else if (l==1)
     ASSIGN(rendered, PySequence_GetItem(rendered,0));
   else
-    ASSIGN(rendered, PyObject_CallFunction(join,"OO",rendered,py_));
+    ASSIGN(rendered, _join_unicode(rendered));
 
   return rendered;
 
@@ -852,11 +875,65 @@ safe_callable(PyObject *self, PyObject *args)
     return PyInt_FromLong(1);
   else
     return PyInt_FromLong(0);
-}  
+}
+
+static PyObject *
+_join_unicode(PyObject *prejoin)
+{
+    PyObject *joined;
+    joined = PyObject_CallFunction(join,"OO",prejoin,py_);
+    if(!joined && PyErr_ExceptionMatches(PyExc_UnicodeError))
+    {
+        int i,l;
+        PyObject *list;
+        PyErr_Clear();
+        list = PySequence_List(prejoin);
+        if(!list)
+        {
+            return NULL;
+        }
+        l = PyList_Size(list);
+        for(i=0;i<l;++i)
+        {
+            PyObject *item = PyList_GetItem(list,i);
+            if(PyString_Check(item))
+            {
+                PyObject *unicode = PyUnicode_DecodeLatin1(PyString_AsString(item),PyString_Size(item),NULL);
+                if(unicode)
+                {
+                    PyList_SetItem(list,i,unicode);
+                }
+                else
+                {
+                    Py_DECREF(list);
+                    return NULL;
+                }
+           }
+       }
+       joined = PyObject_CallFunction(join,"OO",list,py_);
+       Py_DECREF(list);
+    }
+    return joined;
+}
+
+static PyObject *
+join_unicode(PyObject *self, PyObject *args)
+{
+  PyObject *ob;
+
+  UNLESS(PyArg_ParseTuple(args,"O", &ob)) return NULL;
+  return _join_unicode(ob);
+}
+
 
 static struct PyMethodDef Module_Level__methods[] = {
   {"render_blocks", (PyCFunction)render_blocks,	METH_VARARGS,
    ""},
+  {"join_unicode", (PyCFunction)join_unicode,	METH_VARARGS,
+   "join a list of plain strings into a single plain string,"
+   "a list of unicode strings into a single unicode strings,"
+   "or a list containing a mix into a single unicode string with"
+   "the plain strings converted from latin-1"},
   {"safe_callable", (PyCFunction)safe_callable,	METH_VARARGS,
    "callable() with a workaround for a problem with ExtensionClasses\n"
    "and __call__()."},
@@ -871,6 +948,8 @@ initcDocumentTemplate(void)
   DictInstanceType.ob_type=&PyType_Type;
 
   UNLESS (html_quote = PyImport_ImportModule("html_quote")) return;
+  ASSIGN(ustr, PyObject_GetAttrString(html_quote, "ustr"));
+  UNLESS (ustr) return;
   ASSIGN(html_quote, PyObject_GetAttrString(html_quote, "html_quote"));
   UNLESS (html_quote) return;
 

html_quote.py

@@ -1,7 +1,8 @@
 # split off into its own module for aliasing without circrefs
 
 from cgi import escape
+from ustr import ustr
 
 def html_quote(v, name='(Unknown name)', md={}):
-    return escape(str(v), 1)
+    return escape(ustr(v), 1)
 

pDocumentTemplate.py

@@ -13,8 +13,8 @@
 __doc__='''Python implementations of document template some features
 
 
-$Id: pDocumentTemplate.py,v 1.34 2002/02/07 17:47:42 andreasjung Exp $'''
-__version__='$Revision: 1.34 $'[11:-2]
+$Id: pDocumentTemplate.py,v 1.35 2002/03/27 10:14:02 htrd Exp $'''
+__version__='$Revision: 1.35 $'[11:-2]
 
 import  sys, types
 
@@ -41,6 +41,7 @@ def safe_callable(ob):
 
 
 StringType=type('')
+UnicodeType=type(u'')
 TupleType=type(())
 
 
@@ -187,7 +188,7 @@ def render_blocks(blocks, md):
                 section=section[0]
                 if type(section) is StringType: section=md[section]
                 else: section=section(md)
-                section=str(section)
+                section=ustr(section)
             else:
                 # if
                 cache={}
@@ -220,13 +221,29 @@ def render_blocks(blocks, md):
 
                 finally: md._pop()
 
-        elif type(section) is not StringType:
+        elif type(section) is not StringType and type(section) is not UnicodeType:
             section=section(md)
 
         if section: rendered.append(section)
 
     l=len(rendered)
     if l==0: return ''
     elif l==1: return rendered[0]
-    return ''.join(rendered)
-    return rendered
+    return join_unicode(rendered)
+
+def join_unicode(rendered):
+    """join a list of plain strings into a single plain string,
+    a list of unicode strings into a single unicode strings,
+    or a list containing a mix into a single unicode string with
+    the plain strings converted from latin-1
+    """
+    try:
+        return ''.join(rendered)
+    except UnicodeError:
+        # A mix of unicode string and non-ascii plain strings.
+        # Fix up the list, treating normal strings as latin-1
+        rendered = list(rendered)
+        for i in range(len(rendered)):
+            if type(rendered[i]) is StringType:
+                rendered[i] = unicode(rendered[i],'latin-1')
+        return u''.join(rendered)

tests/testDTMLUnicode.py

@@ -0,0 +1,93 @@
+##############################################################################
+#
+# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
+# 
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+# 
+##############################################################################
+"""Document Template Tests
+"""
+
+__rcs_id__='$Id: testDTMLUnicode.py,v 1.2 2002/03/27 10:14:02 htrd Exp $'
+__version__='$Revision: 1.2 $'[11:-2]
+
+import sys, os
+import unittest
+
+from DocumentTemplate.DT_HTML import HTML, String
+from ExtensionClass import Base
+
+class force_str:
+    # A class whose string representation is not always a plain string:
+    def __init__(self,s):
+        self.s = s
+    def __str__(self):
+        return self.s
+
+class DTMLUnicodeTests (unittest.TestCase):
+
+    doc_class = HTML
+
+    def testAA(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = 'helloworld'
+        res = html(a='hello',b='world')
+        assert res == expected, `res`
+
+    def testUU(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'helloworld'
+        res = html(a=u'hello',b=u'world')
+        assert res == expected, `res`
+
+    def testAU(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'helloworld'
+        res = html(a='hello',b=u'world')
+        assert res == expected, `res`
+
+    def testAB(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = 'hello\xc8'
+        res = html(a='hello',b=chr(200))
+        assert res == expected, `res`
+
+    def testUB(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'hello\xc8'
+        res = html(a=u'hello',b=chr(200))
+        assert res == expected, `res`
+
+    def testUB2(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'\u07d0\xc8'
+        res = html(a=unichr(2000),b=chr(200))
+        assert res == expected, `res`
+
+    def testUnicodeStr(self):
+        html=self.doc_class('<dtml-var a><dtml-var b>')
+        expected = u'\u07d0\xc8'
+        res = html(a=force_str(unichr(2000)),b=chr(200))
+        assert res == expected, `res`
+
+    def testUqB(self):
+        html=self.doc_class('<dtml-var a html_quote><dtml-var b>')
+        expected = u'he&gt;llo\xc8'
+        res = html(a=u'he>llo',b=chr(200))
+        assert res == expected, `res`
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest( unittest.makeSuite( DTMLUnicodeTests ) )
+    return suite
+
+def main():
+    unittest.TextTestRunner().run(test_suite())
+
+if __name__ == '__main__':
+    main()