Skip to content
Sign up
This repository

public / yuidoc Octocat-spinner-32

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Completely refactored all io operations to eliminate anything encodin… 

…g unicode to utf-8. Fixes unicode battles with the latest versions of Cheetah.
  • Loading branch information...
commit c4382388aed6188dffc6e4e5d67bbfb198a6cec6 1 parent 2affa9a
authored

Showing 6 changed files with 40 additions and 59 deletions. Show Diff Stats Hide Diff Stats

  1. 14  INSTALL
  2. 10  KNOWN_ISSUES
  3. 42  bin/yuidoc_generate.py
  4. 5  bin/yuidoc_highlight.py
  5. 24  bin/yuidoc_parse.py
  6. 4  template/main.tmpl
14  INSTALL
View 
@@ -25,20 +25,6 @@ Unix (including cygwin):
25 25
26 26 
     easy_install pygments
27 27 
     easy_install Cheetah
28   
-
29   
-    *****************************************************************************
30   
-    *****************************************************************************
31   
-    ***
32   
-    *** NOTE: recent versions of Cheetah are sometimes throwing errors when
33   
-    *** encountering international characters.  Until this is resolved, you must
34   
-    *** use Cheetah 2.1.0 or below if you are experiencing a unicode error
35   
-    *** when parsing your files.
36   
-    ***
37   
-    *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
38   
-    ***
39   
-    *****************************************************************************
40   
-    *****************************************************************************
41   
-
42 28 
     easy_install simplejson
43 29
44 30
10  KNOWN_ISSUES
View
... ... 
@@ -1,14 +1,4 @@
1 1 
 **************************************************************************
2   
-Unicode Errors:
3   
-
4   
-Recent versions of Cheetah are sometimes throwing errors when
5   
-encountering international characters.  Until this is resolved, you must
6   
-use Cheetah 2.1.0 or below if you are experiencing unicode errors
7   
-when parsing your files.
8   
-
9   
-easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
10   
-
11   
-**************************************************************************
12 2 
 Mapping Files to Modules:
13 3
14 4 
 Because a single @module is supported for a set of files, the parser expects that all files in
42  bin/yuidoc_generate.py
View 
@@ -12,9 +12,10 @@
12 12 
 ''' Prints documentation with htmltmpl from the json data outputted by parser.py  '''
13 13 
 import os, re, simplejson, shutil, logging, logging.config, time, datetime
14 14 
 from const import *
15   
-from cStringIO import StringIO
  15 
+# from cStringIO import StringIO
16 16 
 from Cheetah.Template import Template
17 17 
 from sets import Set
  18 
+import codecs
18 19
19 20 
 try:
20 21 
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
 
@@ -68,8 +69,11 @@ def _mkdir(newdir):
68 69
69 70 
         self.showprivate  = showprivate
70 71
71   
-        f=open(os.path.join(inpath, datafile))
72   
-        self.rawdata = StringIO(f.read()).getvalue()
  72 
+        f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" )
  73 
+        self.rawdata = f.read()
  74 
+
  75 
+        # log.info('INPUT DATA: ' + self.rawdata)
  76 
+
73 77 
         d = self.data = simplejson.loads(self.rawdata)
74 78
75 79 
         self.projectname = projectname
 
@@ -102,11 +106,15 @@ def cleanseStr(self, strg):
102 106 
         # log.warn('cleansed module: %s' %(cleansed));
103 107 
         return self.moduleprefix + cleansed
104 108
105   
-    def write(self, filename, data):
106   
-        out = open(os.path.join(self.outpath, filename), "w")
107   
-        out.writelines(str(data))
108   
-        # out.writelines(unicode(data))
109   
-        # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace'))
  109 
+    def write(self, filename, data, template=True):
  110 
+        out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" )
  111 
+
  112 
+        if template:
  113 
+            datastr = data.respond()
  114 
+            out.write(datastr)
  115 
+        else:
  116 
+            out.write(data)
  117 
+
110 118 
         out.close()
111 119
112 120 
     def process(self):
 
@@ -129,7 +137,7 @@ def assignGlobalProperties(template):
129 137 
             template.filename     = self.filename
130 138 
             if self.filename:
131 139 
                 template.filepath = os.path.join(self.inpath, self.filename)
132   
-                template.filepath_highlighted = template.filepath + self.newext
  140 
+                template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read()
133 141
134 142 
             template.pagetype     = self.pagetype
135 143 
             template.classmap     = self.classmap
 
@@ -267,7 +275,7 @@ def allprop_sort(x, y):
267 275 
         # jsonname = self.cleansedmodulename + ".json"
268 276 
         jsonname = "raw.json"
269 277 
         log.info("Writing " + jsonname)
270   
-        self.write(jsonname, self.rawdata)
  278 
+        self.write(jsonname, self.rawdata, False)
271 279
272 280 
         for mname in self.modules:
273 281 
             log.info("Generating module splash for %s" %(mname))
 
@@ -380,6 +388,7 @@ def allprop_sort(x, y):
380 388 
                             if self.showprivate or PRIVATE not in prop:
381 389 
                                 propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)}
382 390
  391 
+
383 392 
                                 transferToDict( ACCESS,   prop, propdata           )
384 393 
                                 if PRIVATE in prop: propdata[ACCESS] = PRIVATE
385 394 
                                 elif PROTECTED in prop: propdata[ACCESS] = PROTECTED
 
@@ -398,6 +407,7 @@ def allprop_sort(x, y):
398 407 
                                 if FINAL in prop: propdata[FINAL] = FINAL
399 408 
                                 props.append(propdata)
400 409
  410 
+
401 411 
                     # Methods
402 412 
                     methods = t.methods = []
403 413 
                     if METHODS in c:
 
@@ -574,11 +584,11 @@ def allprop_sort(x, y):
574 584 
                     # write module splash
575 585 
                     moduleprops.sort(allprop_sort)
576 586 
                     t.allprops_raw = moduleprops
577   
-                    moduleprops_json =  simplejson.dumps(moduleprops)
  587 
+                    moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
578 588 
                     t.allprops = moduleprops_json
579 589 
                     classList.sort(allprop_sort)
580 590 
                     t.classList_raw = classList
581   
-                    t.classList = simplejson.dumps(classList)
  591 
+                    t.classList = simplejson.dumps(classList, ensure_ascii=False)
582 592 
                     self.write("%s.html" %(self.classname), t)
583 593
584 594 
             # clear out class name
 
@@ -594,7 +604,7 @@ def allprop_sort(x, y):
594 604 
             # write module splash
595 605 
             moduleprops.sort(allprop_sort)
596 606 
             t.allprops_raw = moduleprops
597   
-            moduleprops_json =  simplejson.dumps(moduleprops)
  607 
+            moduleprops_json =  simplejson.dumps(moduleprops, ensure_ascii=False)
598 608 
             t.allprops = moduleprops_json
599 609
600 610 
             # log.warn('cleansed module file name: %s' %(t.cleansedmodulename));
 
@@ -619,8 +629,8 @@ def allprop_sort(x, y):
619 629
620 630 
         allprops.sort(allprop_sort)
621 631
622   
-        allprops_json =  simplejson.dumps(allprops)
623   
-        self.write("index.json",allprops_json)
  632 
+        allprops_json =  simplejson.dumps(allprops, ensure_ascii=False)
  633 
+        self.write("index.json", allprops_json, False)
624 634
625 635 
         # index
626 636 
         log.info("Generating index")
 
@@ -666,7 +676,7 @@ def allprop_sort(x, y):
666 676 
                     log.warn('class map ' + i + ' failure (no module declaration?)')
667 677 
                 except: pass
668 678
669   
-        t.pkgmap = simplejson.dumps(pkgMap)
  679 
+        t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False)
670 680 
         self.write("classmap.js", t)
671 681
672 682
5  bin/yuidoc_highlight.py
View 
@@ -11,7 +11,6 @@
11 11
12 12 
 import os, re, string, logging, logging.config
13 13 
 from const import *
14   
-from cStringIO import StringIO
15 14 
 from optparse import OptionParser
16 15 
 from pygments import highlight
17 16 
 from pygments.formatters import HtmlFormatter
 
@@ -76,8 +75,8 @@ def highlightFile(path, file):
76 75
77 76 
             highlighted = highlightString(fileStr)
78 77
79   
-            out = open(os.path.join(self.outputdir, file + self.newext), "w")
80   
-            out.writelines(highlighted.encode('utf-8'))
  78 
+            out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" )
  79 
+            out.write(highlighted)
81 80 
             out.close()
82 81
83 82 
         def highlightDir(path):
24  bin/yuidoc_parse.py
View 
@@ -13,8 +13,9 @@
13 13 
     an API. It is designed to parse one module at a time '''
14 14 
 import os, re, simplejson, string, sys, pprint, logging, logging.config
15 15 
 from const import *
16   
-from cStringIO import StringIO
  16 
+# from cStringIO import StringIO
17 17 
 from optparse import OptionParser
  18 
+import codecs
18 19
19 20 
 try:
20 21 
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
 
@@ -39,18 +40,21 @@ def _mkdir(newdir):
39 40 
                 if tail: os.mkdir(newdir)
40 41
41 42 
         def parseFile(path, file):
42   
-            f=open(os.path.join(path, file))
  43 
+            # f=open(os.path.join(path, file))
  44 
+            f = codecs.open(os.path.join(path, file), "r", "utf-8" )
43 45 
             #adding a return to the beginning of the line allows for:
44 46 
             #  #!/usr/bin/foo
45 47 
             #  MOVED to the file marker
46 48 
             # fileStr = StringIO("\n%s" % f.read()).getvalue()
47   
-            fileStr = StringIO("%s" % f.read()).getvalue()
  49 
+            # fileStr = StringIO("%s" % f.read()).getvalue()
  50 
+            fileStr = f.read()
48 51 
             log.info("parsing " + file)
49 52 
             # add a file marker token so the parser can keep track of what is in what file
50 53 
             content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file)
51 54
52 55 
             # copy
53   
-            out = open(os.path.join(self.outputdir, file), "w")
  56 
+            # out = open(os.path.join(self.outputdir, file), "w")
  57 
+            out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" )
54 58 
             out.writelines(fileStr)
55 59 
             out.close()
56 60 
 
@@ -142,9 +146,9 @@ def parseDir(path):
142 146 
                 self.parse(self.tokenize(match))
143 147
144 148
145   
-        out = open(os.path.join(self.outputdir, outputfile), "w")
  149 
+        out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" )
146 150
147   
-        out.writelines(simplejson.dumps(self.data))
  151 
+        out.writelines(simplejson.dumps(self.data, ensure_ascii=False))
148 152 
         out.close()
149 153
150 154 
 
@@ -325,15 +329,11 @@ def parseParams(tokenMap, dict, srctag=PARAM, desttag=PARAMS):
325 329 
                         log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap)))
326 330 
                         sys.exit()
327 331
328   
-                    # description.encode('utf-8', 'xmlcharrefreplace')
329   
-                    # description = unicode(description, 'utf-8', 'xmlcharrefreplace')
330   
-
331   
-
332 332 
                     mo = self.param_pat.match(description)
333 333 
                     if mo:
334 334 
                         name = mo.group(1)
335 335 
                         description = mo.group(2)
336   
-                        description.encode('utf-8', 'xmlcharrefreplace')
  336 
+                        # description.encode('utf-8', 'xmlcharrefreplace')
337 337
338 338 
                         dict[desttag].append({
339 339 
                                 NAME:        name,
 
@@ -436,8 +436,6 @@ def defineClass(name):
436 436 
                 # for the block
437 437 
                 if token and DESCRIPTION not in tokenMap:
438 438
439   
-                    # token = unicode(token, 'utf-8', 'xmlcharrefreplace')
440   
-                    ############################ token.encode('utf-8', 'xmlcharrefreplace')
441 439 
                     tokenMap[DESCRIPTION] = [token]
442 440 
                 else: pass # I don't think this can happen any longer
443 441
4  template/main.tmpl
View
... ... 
@@ -1,5 +1,4 @@
1 1 
 #encoding UTF-8
2   
-#filter EncodeUnicode
3 2 
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
4 3 
 <html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#">
5 4 
 <head>
 
@@ -61,7 +60,7 @@
61 60 
                         <style>
62 61 
                             #doc3 .classopts { display:none; }
63 62 
                         </style>
64   
-                        #include raw $filepath_highlighted
  63 
+                        $highlightcontent
65 64 
                     </div>
66 65 
                 #else if $classname
67 66 
                     <h2>
 
@@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit();
687 686 
 #end if
688 687 
 </body>
689 688 
 </html>
690   
-#end filter

0 notes on commit c438238

Please sign in to comment.
Something went wrong with that request. Please try again.