yui / yuidoc

0
@@ -25,20 +25,6 @@ Unix (including cygwin):
0
 
0
     easy_install pygments
0
     easy_install Cheetah
0
-
0
-    *****************************************************************************
0
-    *****************************************************************************
0
-    ***
0
-    *** NOTE: recent versions of Cheetah are sometimes throwing errors when
0
-    *** encountering international characters.  Until this is resolved, you must
0
-    *** use Cheetah 2.1.0 or below if you are experiencing a unicode error
0
-    *** when parsing your files.
0
-    ***
0
-    *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
0
-    ***
0
-    *****************************************************************************
0
-    *****************************************************************************
0
-
0
     easy_install simplejson
0
 
0
 

0
@@ -1,14 +1,4 @@
0
 **************************************************************************
0
-Unicode Errors:
0
-
0
-Recent versions of Cheetah are sometimes throwing errors when
0
-encountering international characters.  Until this is resolved, you must
0
-use Cheetah 2.1.0 or below if you are experiencing unicode errors
0
-when parsing your files.
0
-
0
-easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
0
-
0
-**************************************************************************
0
 Mapping Files to Modules:
0
 
0
 Because a single @module is supported for a set of files, the parser expects that all files in 

0
@@ -12,9 +12,10 @@ version: 1.0.0b1
0
 ''' Prints documentation with htmltmpl from the json data outputted by parser.py  ''' 
0
 import os, re, simplejson, shutil, logging, logging.config, time, datetime
0
 from const import *
0
-from cStringIO import StringIO 
0
+# from cStringIO import StringIO 
0
 from Cheetah.Template import Template
0
 from sets import Set
0
+import codecs
0
 
0
 try:
0
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
0
@@ -68,8 +69,11 @@ class DocGenerator(object):
0
 
0
         self.showprivate  = showprivate
0
 
0
-        f=open(os.path.join(inpath, datafile))
0
-        self.rawdata = StringIO(f.read()).getvalue()
0
+        f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" )
0
+        self.rawdata = f.read()
0
+
0
+        # log.info('INPUT DATA: ' + self.rawdata)
0
+
0
         d = self.data = simplejson.loads(self.rawdata)
0
 
0
         self.projectname = projectname
0
@@ -102,11 +106,15 @@ class DocGenerator(object):
0
         # log.warn('cleansed module: %s' %(cleansed));
0
         return self.moduleprefix + cleansed
0
 
0
-    def write(self, filename, data):
0
-        out = open(os.path.join(self.outpath, filename), "w")
0
-        out.writelines(str(data))
0
-        # out.writelines(unicode(data))
0
-        # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace'))
0
+    def write(self, filename, data, template=True):
0
+        out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" )
0
+
0
+        if template:
0
+            datastr = data.respond()
0
+            out.write(datastr)
0
+        else:
0
+            out.write(data)
0
+
0
         out.close()
0
 
0
     def process(self):
0
@@ -129,7 +137,7 @@ class DocGenerator(object):
0
             template.filename     = self.filename
0
             if self.filename:
0
                 template.filepath = os.path.join(self.inpath, self.filename)
0
-                template.filepath_highlighted = template.filepath + self.newext
0
+                template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read()
0
 
0
             template.pagetype     = self.pagetype
0
             template.classmap     = self.classmap
0
@@ -267,7 +275,7 @@ class DocGenerator(object):
0
         # jsonname = self.cleansedmodulename + ".json"
0
         jsonname = "raw.json"
0
         log.info("Writing " + jsonname)
0
-        self.write(jsonname, self.rawdata)
0
+        self.write(jsonname, self.rawdata, False)
0
 
0
         for mname in self.modules:
0
             log.info("Generating module splash for %s" %(mname))
0
@@ -380,6 +388,7 @@ class DocGenerator(object):
0
                             if self.showprivate or PRIVATE not in prop:
0
                                 propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)}
0
 
0
+
0
                                 transferToDict( ACCESS,   prop, propdata           )
0
                                 if PRIVATE in prop: propdata[ACCESS] = PRIVATE
0
                                 elif PROTECTED in prop: propdata[ACCESS] = PROTECTED
0
@@ -398,6 +407,7 @@ class DocGenerator(object):
0
                                 if FINAL in prop: propdata[FINAL] = FINAL
0
                                 props.append(propdata)
0
 
0
+
0
                     # Methods
0
                     methods = t.methods = []
0
                     if METHODS in c:
0
@@ -574,11 +584,11 @@ class DocGenerator(object):
0
                     # write module splash
0
                     moduleprops.sort(allprop_sort)
0
                     t.allprops_raw = moduleprops
0
-                    moduleprops_json =  simplejson.dumps(moduleprops)
0
+                    moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
0
                     t.allprops = moduleprops_json
0
                     classList.sort(allprop_sort)
0
                     t.classList_raw = classList
0
-                    t.classList = simplejson.dumps(classList)
0
+                    t.classList = simplejson.dumps(classList, ensure_ascii=False)
0
                     self.write("%s.html" %(self.classname), t)
0
         
0
             # clear out class name
0
@@ -594,7 +604,7 @@ class DocGenerator(object):
0
             # write module splash
0
             moduleprops.sort(allprop_sort)
0
             t.allprops_raw = moduleprops
0
-            moduleprops_json =  simplejson.dumps(moduleprops)
0
+            moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
0
             t.allprops = moduleprops_json
0
 
0
             # log.warn('cleansed module file name: %s' %(t.cleansedmodulename));
0
@@ -619,8 +629,8 @@ class DocGenerator(object):
0
 
0
         allprops.sort(allprop_sort)
0
                                             
0
-        allprops_json =  simplejson.dumps(allprops)
0
-        self.write("index.json",allprops_json)
0
+        allprops_json =  simplejson.dumps(allprops, ensure_ascii=False)
0
+        self.write("index.json", allprops_json, False)
0
 
0
         # index
0
         log.info("Generating index")
0
@@ -666,7 +676,7 @@ class DocGenerator(object):
0
                     log.warn('class map ' + i + ' failure (no module declaration?)')
0
                 except: pass
0
 
0
-        t.pkgmap = simplejson.dumps(pkgMap)
0
+        t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False)
0
         self.write("classmap.js", t)
0
 
0
 

0
@@ -11,7 +11,6 @@ version: 1.0.0b1
0
 
0
 import os, re, string, logging, logging.config
0
 from const import *
0
-from cStringIO import StringIO 
0
 from optparse import OptionParser
0
 from pygments import highlight
0
 from pygments.formatters import HtmlFormatter
0
@@ -76,8 +75,8 @@ class DocHighlighter(object):
0
 
0
             highlighted = highlightString(fileStr)
0
 
0
-            out = open(os.path.join(self.outputdir, file + self.newext), "w")
0
-            out.writelines(highlighted.encode('utf-8'))
0
+            out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" )
0
+            out.write(highlighted)
0
             out.close()
0
 
0
         def highlightDir(path):

0
@@ -13,8 +13,9 @@ version: 1.0.0b1
0
     an API. It is designed to parse one module at a time ''' 
0
 import os, re, simplejson, string, sys, pprint, logging, logging.config
0
 from const import *
0
-from cStringIO import StringIO 
0
+# from cStringIO import StringIO 
0
 from optparse import OptionParser
0
+import codecs
0
 
0
 try:
0
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
0
@@ -39,18 +40,21 @@ class DocParser(object):
0
                 if tail: os.mkdir(newdir)
0
 
0
         def parseFile(path, file):
0
-            f=open(os.path.join(path, file))
0
+            # f=open(os.path.join(path, file))
0
+            f = codecs.open(os.path.join(path, file), "r", "utf-8" )
0
             #adding a return to the beginning of the line allows for:
0
             #  #!/usr/bin/foo
0
             #  MOVED to the file marker
0
             # fileStr = StringIO("\n%s" % f.read()).getvalue()
0
-            fileStr = StringIO("%s" % f.read()).getvalue()
0
+            # fileStr = StringIO("%s" % f.read()).getvalue()
0
+            fileStr = f.read()
0
             log.info("parsing " + file)
0
             # add a file marker token so the parser can keep track of what is in what file
0
             content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file)
0
 
0
             # copy
0
-            out = open(os.path.join(self.outputdir, file), "w")
0
+            # out = open(os.path.join(self.outputdir, file), "w")
0
+            out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" )
0
             out.writelines(fileStr)
0
             out.close()
0
 
0
@@ -142,9 +146,9 @@ class DocParser(object):
0
                 self.parse(self.tokenize(match))
0
 
0
         
0
-        out = open(os.path.join(self.outputdir, outputfile), "w")
0
+        out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" )
0
 
0
-        out.writelines(simplejson.dumps(self.data))
0
+        out.writelines(simplejson.dumps(self.data, ensure_ascii=False))
0
         out.close()
0
 
0
 
0
@@ -325,15 +329,11 @@ class DocParser(object):
0
                         log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap)))
0
                         sys.exit()
0
 
0
-                    # description.encode('utf-8', 'xmlcharrefreplace')
0
-                    # description = unicode(description, 'utf-8', 'xmlcharrefreplace')
0
-
0
-
0
                     mo = self.param_pat.match(description)
0
                     if mo:
0
                         name = mo.group(1)
0
                         description = mo.group(2)
0
-                        description.encode('utf-8', 'xmlcharrefreplace')
0
+                        # description.encode('utf-8', 'xmlcharrefreplace')
0
 
0
                         dict[desttag].append({  
0
                                 NAME:        name,
0
@@ -436,8 +436,6 @@ it was empty" % token
0
                 # for the block
0
                 if token and DESCRIPTION not in tokenMap:
0
 
0
-                    # token = unicode(token, 'utf-8', 'xmlcharrefreplace')
0
-                    ############################ token.encode('utf-8', 'xmlcharrefreplace')
0
                     tokenMap[DESCRIPTION] = [token]
0
                 else: pass # I don't think this can happen any longer
0
 

0
@@ -1,5 +1,4 @@
0
 #encoding UTF-8
0
-#filter EncodeUnicode
0
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
0
 <html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#">
0
 <head>
0
@@ -61,7 +60,7 @@
0
                         <style>
0
                             #doc3 .classopts { display:none; }
0
                         </style>
0
-                        #include raw $filepath_highlighted
0
+                        $highlightcontent
0
                     </div>
0
                 #else if $classname 
0
                     <h2>
0
@@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit();
0
 #end if
0
 </body>
0
 </html>
0
-#end filter