Completely refactored all io operations to eliminate anything encodin…

…g unicode to utf-8. Fixes unicode battles with the latest versions of Cheetah.

INSTALL

@@ -25,20 +25,6 @@ Unix (including cygwin):
 
     easy_install pygments
     easy_install Cheetah
-
-    *****************************************************************************
-    *****************************************************************************
-    ***
-    *** NOTE: recent versions of Cheetah are sometimes throwing errors when
-    *** encountering international characters.  Until this is resolved, you must
-    *** use Cheetah 2.1.0 or below if you are experiencing a unicode error
-    *** when parsing your files.
-    ***
-    *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
-    ***
-    *****************************************************************************
-    *****************************************************************************
-
     easy_install simplejson
 
 

KNOWN_ISSUES

@@ -1,13 +1,3 @@
-**************************************************************************
-Unicode Errors:
-
-Recent versions of Cheetah are sometimes throwing errors when
-encountering international characters.  Until this is resolved, you must
-use Cheetah 2.1.0 or below if you are experiencing unicode errors
-when parsing your files.
-
-easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
-
 **************************************************************************
 Mapping Files to Modules:
 

bin/yuidoc_generate.py

@@ -12,9 +12,10 @@
 ''' Prints documentation with htmltmpl from the json data outputted by parser.py  ''' 
 import os, re, simplejson, shutil, logging, logging.config, time, datetime
 from const import *
-from cStringIO import StringIO 
+# from cStringIO import StringIO 
 from Cheetah.Template import Template
 from sets import Set
+import codecs
 
 try:
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -68,8 +69,11 @@ def _mkdir(newdir):
 
         self.showprivate  = showprivate
 
-        f=open(os.path.join(inpath, datafile))
-        self.rawdata = StringIO(f.read()).getvalue()
+        f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" )
+        self.rawdata = f.read()
+
+        # log.info('INPUT DATA: ' + self.rawdata)
+
         d = self.data = simplejson.loads(self.rawdata)
 
         self.projectname = projectname
@@ -102,11 +106,15 @@ def cleanseStr(self, strg):
         # log.warn('cleansed module: %s' %(cleansed));
         return self.moduleprefix + cleansed
 
-    def write(self, filename, data):
-        out = open(os.path.join(self.outpath, filename), "w")
-        out.writelines(str(data))
-        # out.writelines(unicode(data))
-        # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace'))
+    def write(self, filename, data, template=True):
+        out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" )
+
+        if template:
+            datastr = data.respond()
+            out.write(datastr)
+        else:
+            out.write(data)
+
         out.close()
 
     def process(self):
@@ -129,7 +137,7 @@ def assignGlobalProperties(template):
             template.filename     = self.filename
             if self.filename:
                 template.filepath = os.path.join(self.inpath, self.filename)
-                template.filepath_highlighted = template.filepath + self.newext
+                template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read()
 
             template.pagetype     = self.pagetype
             template.classmap     = self.classmap
@@ -267,7 +275,7 @@ def allprop_sort(x, y):
         # jsonname = self.cleansedmodulename + ".json"
         jsonname = "raw.json"
         log.info("Writing " + jsonname)
-        self.write(jsonname, self.rawdata)
+        self.write(jsonname, self.rawdata, False)
 
         for mname in self.modules:
             log.info("Generating module splash for %s" %(mname))
@@ -380,6 +388,7 @@ def allprop_sort(x, y):
                             if self.showprivate or PRIVATE not in prop:
                                 propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)}
 
+
                                 transferToDict( ACCESS,   prop, propdata           )
                                 if PRIVATE in prop: propdata[ACCESS] = PRIVATE
                                 elif PROTECTED in prop: propdata[ACCESS] = PROTECTED
@@ -398,6 +407,7 @@ def allprop_sort(x, y):
                                 if FINAL in prop: propdata[FINAL] = FINAL
                                 props.append(propdata)
 
+
                     # Methods
                     methods = t.methods = []
                     if METHODS in c:
@@ -574,11 +584,11 @@ def allprop_sort(x, y):
                     # write module splash
                     moduleprops.sort(allprop_sort)
                     t.allprops_raw = moduleprops
-                    moduleprops_json =  simplejson.dumps(moduleprops)
+                    moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
                     t.allprops = moduleprops_json
                     classList.sort(allprop_sort)
                     t.classList_raw = classList
-                    t.classList = simplejson.dumps(classList)
+                    t.classList = simplejson.dumps(classList, ensure_ascii=False)
                     self.write("%s.html" %(self.classname), t)
         
             # clear out class name
@@ -594,7 +604,7 @@ def allprop_sort(x, y):
             # write module splash
             moduleprops.sort(allprop_sort)
             t.allprops_raw = moduleprops
-            moduleprops_json =  simplejson.dumps(moduleprops)
+            moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
             t.allprops = moduleprops_json
 
             # log.warn('cleansed module file name: %s' %(t.cleansedmodulename));
@@ -619,8 +629,8 @@ def allprop_sort(x, y):
 
         allprops.sort(allprop_sort)
                                             
-        allprops_json =  simplejson.dumps(allprops)
-        self.write("index.json",allprops_json)
+        allprops_json =  simplejson.dumps(allprops, ensure_ascii=False)
+        self.write("index.json", allprops_json, False)
 
         # index
         log.info("Generating index")
@@ -666,7 +676,7 @@ def allprop_sort(x, y):
                     log.warn('class map ' + i + ' failure (no module declaration?)')
                 except: pass
 
-        t.pkgmap = simplejson.dumps(pkgMap)
+        t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False)
         self.write("classmap.js", t)
 
 

bin/yuidoc_highlight.py

@@ -11,7 +11,6 @@
 
 import os, re, string, logging, logging.config
 from const import *
-from cStringIO import StringIO 
 from optparse import OptionParser
 from pygments import highlight
 from pygments.formatters import HtmlFormatter
@@ -76,8 +75,8 @@ def highlightFile(path, file):
 
             highlighted = highlightString(fileStr)
 
-            out = open(os.path.join(self.outputdir, file + self.newext), "w")
-            out.writelines(highlighted.encode('utf-8'))
+            out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" )
+            out.write(highlighted)
             out.close()
 
         def highlightDir(path):

bin/yuidoc_parse.py

@@ -13,8 +13,9 @@
     an API. It is designed to parse one module at a time ''' 
 import os, re, simplejson, string, sys, pprint, logging, logging.config
 from const import *
-from cStringIO import StringIO 
+# from cStringIO import StringIO 
 from optparse import OptionParser
+import codecs
 
 try:
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -39,18 +40,21 @@ def _mkdir(newdir):
                 if tail: os.mkdir(newdir)
 
         def parseFile(path, file):
-            f=open(os.path.join(path, file))
+            # f=open(os.path.join(path, file))
+            f = codecs.open(os.path.join(path, file), "r", "utf-8" )
             #adding a return to the beginning of the line allows for:
             #  #!/usr/bin/foo
             #  MOVED to the file marker
             # fileStr = StringIO("\n%s" % f.read()).getvalue()
-            fileStr = StringIO("%s" % f.read()).getvalue()
+            # fileStr = StringIO("%s" % f.read()).getvalue()
+            fileStr = f.read()
             log.info("parsing " + file)
             # add a file marker token so the parser can keep track of what is in what file
             content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file)
 
             # copy
-            out = open(os.path.join(self.outputdir, file), "w")
+            # out = open(os.path.join(self.outputdir, file), "w")
+            out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" )
             out.writelines(fileStr)
             out.close()
 
@@ -142,9 +146,9 @@ def parseDir(path):
                 self.parse(self.tokenize(match))
 
         
-        out = open(os.path.join(self.outputdir, outputfile), "w")
+        out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" )
 
-        out.writelines(simplejson.dumps(self.data))
+        out.writelines(simplejson.dumps(self.data, ensure_ascii=False))
         out.close()
 
 
@@ -325,15 +329,11 @@ def parseParams(tokenMap, dict, srctag=PARAM, desttag=PARAMS):
                         log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap)))
                         sys.exit()
 
-                    # description.encode('utf-8', 'xmlcharrefreplace')
-                    # description = unicode(description, 'utf-8', 'xmlcharrefreplace')
-
-
                     mo = self.param_pat.match(description)
                     if mo:
                         name = mo.group(1)
                         description = mo.group(2)
-                        description.encode('utf-8', 'xmlcharrefreplace')
+                        # description.encode('utf-8', 'xmlcharrefreplace')
 
                         dict[desttag].append({  
                                 NAME:        name,
@@ -436,8 +436,6 @@ def defineClass(name):
                 # for the block
                 if token and DESCRIPTION not in tokenMap:
 
-                    # token = unicode(token, 'utf-8', 'xmlcharrefreplace')
-                    ############################ token.encode('utf-8', 'xmlcharrefreplace')
                     tokenMap[DESCRIPTION] = [token]
                 else: pass # I don't think this can happen any longer
 

template/main.tmpl

@@ -1,5 +1,4 @@
 #encoding UTF-8
-#filter EncodeUnicode
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
 <html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#">
 <head>
@@ -61,7 +60,7 @@
                         <style>
                             #doc3 .classopts { display:none; }
                         </style>
-                        #include raw $filepath_highlighted
+                        $highlightcontent
                     </div>
                 #else if $classname 
                     <h2>
@@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit();
 #end if
 </body>
 </html>
-#end filter