Permalink
Browse files

Completely refactored all io operations to eliminate anything encodin…

…g unicode to utf-8. Fixes unicode battles with the latest versions of Cheetah.
  • Loading branch information...
1 parent 2affa9a commit c4382388aed6188dffc6e4e5d67bbfb198a6cec6 Adam Moore committed Jul 3, 2009
Showing with 40 additions and 59 deletions.
  1. +0 −14 INSTALL
  2. +0 −10 KNOWN_ISSUES
  3. +26 −16 bin/yuidoc_generate.py
  4. +2 −3 bin/yuidoc_highlight.py
  5. +11 −13 bin/yuidoc_parse.py
  6. +1 −3 template/main.tmpl
View
14 INSTALL
@@ -25,20 +25,6 @@ Unix (including cygwin):
easy_install pygments
easy_install Cheetah
-
- *****************************************************************************
- *****************************************************************************
- ***
- *** NOTE: recent versions of Cheetah are sometimes throwing errors when
- *** encountering international characters. Until this is resolved, you must
- *** use Cheetah 2.1.0 or below if you are experiencing a unicode error
- *** when parsing your files.
- ***
- *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
- ***
- *****************************************************************************
- *****************************************************************************
-
easy_install simplejson
View
10 KNOWN_ISSUES
@@ -1,14 +1,4 @@
**************************************************************************
-Unicode Errors:
-
-Recent versions of Cheetah are sometimes throwing errors when
-encountering international characters. Until this is resolved, you must
-use Cheetah 2.1.0 or below if you are experiencing unicode errors
-when parsing your files.
-
-easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
-
-**************************************************************************
Mapping Files to Modules:
Because a single @module is supported for a set of files, the parser expects that all files in
View
42 bin/yuidoc_generate.py
@@ -12,9 +12,10 @@
''' Prints documentation with htmltmpl from the json data outputted by parser.py '''
import os, re, simplejson, shutil, logging, logging.config, time, datetime
from const import *
-from cStringIO import StringIO
+# from cStringIO import StringIO
from Cheetah.Template import Template
from sets import Set
+import codecs
try:
logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -68,8 +69,11 @@ def _mkdir(newdir):
self.showprivate = showprivate
- f=open(os.path.join(inpath, datafile))
- self.rawdata = StringIO(f.read()).getvalue()
+ f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" )
+ self.rawdata = f.read()
+
+ # log.info('INPUT DATA: ' + self.rawdata)
+
d = self.data = simplejson.loads(self.rawdata)
self.projectname = projectname
@@ -102,11 +106,15 @@ def cleanseStr(self, strg):
# log.warn('cleansed module: %s' %(cleansed));
return self.moduleprefix + cleansed
- def write(self, filename, data):
- out = open(os.path.join(self.outpath, filename), "w")
- out.writelines(str(data))
- # out.writelines(unicode(data))
- # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace'))
+ def write(self, filename, data, template=True):
+ out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" )
+
+ if template:
+ datastr = data.respond()
+ out.write(datastr)
+ else:
+ out.write(data)
+
out.close()
def process(self):
@@ -129,7 +137,7 @@ def assignGlobalProperties(template):
template.filename = self.filename
if self.filename:
template.filepath = os.path.join(self.inpath, self.filename)
- template.filepath_highlighted = template.filepath + self.newext
+ template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read()
template.pagetype = self.pagetype
template.classmap = self.classmap
@@ -267,7 +275,7 @@ def allprop_sort(x, y):
# jsonname = self.cleansedmodulename + ".json"
jsonname = "raw.json"
log.info("Writing " + jsonname)
- self.write(jsonname, self.rawdata)
+ self.write(jsonname, self.rawdata, False)
for mname in self.modules:
log.info("Generating module splash for %s" %(mname))
@@ -380,6 +388,7 @@ def allprop_sort(x, y):
if self.showprivate or PRIVATE not in prop:
propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)}
+
transferToDict( ACCESS, prop, propdata )
if PRIVATE in prop: propdata[ACCESS] = PRIVATE
elif PROTECTED in prop: propdata[ACCESS] = PROTECTED
@@ -398,6 +407,7 @@ def allprop_sort(x, y):
if FINAL in prop: propdata[FINAL] = FINAL
props.append(propdata)
+
# Methods
methods = t.methods = []
if METHODS in c:
@@ -574,11 +584,11 @@ def allprop_sort(x, y):
# write module splash
moduleprops.sort(allprop_sort)
t.allprops_raw = moduleprops
- moduleprops_json = simplejson.dumps(moduleprops)
+ moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False)
t.allprops = moduleprops_json
classList.sort(allprop_sort)
t.classList_raw = classList
- t.classList = simplejson.dumps(classList)
+ t.classList = simplejson.dumps(classList, ensure_ascii=False)
self.write("%s.html" %(self.classname), t)
# clear out class name
@@ -594,7 +604,7 @@ def allprop_sort(x, y):
# write module splash
moduleprops.sort(allprop_sort)
t.allprops_raw = moduleprops
- moduleprops_json = simplejson.dumps(moduleprops)
+ moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False)
t.allprops = moduleprops_json
# log.warn('cleansed module file name: %s' %(t.cleansedmodulename));
@@ -619,8 +629,8 @@ def allprop_sort(x, y):
allprops.sort(allprop_sort)
- allprops_json = simplejson.dumps(allprops)
- self.write("index.json",allprops_json)
+ allprops_json = simplejson.dumps(allprops, ensure_ascii=False)
+ self.write("index.json", allprops_json, False)
# index
log.info("Generating index")
@@ -666,7 +676,7 @@ def allprop_sort(x, y):
log.warn('class map ' + i + ' failure (no module declaration?)')
except: pass
- t.pkgmap = simplejson.dumps(pkgMap)
+ t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False)
self.write("classmap.js", t)
View
5 bin/yuidoc_highlight.py
@@ -11,7 +11,6 @@
import os, re, string, logging, logging.config
from const import *
-from cStringIO import StringIO
from optparse import OptionParser
from pygments import highlight
from pygments.formatters import HtmlFormatter
@@ -76,8 +75,8 @@ def highlightFile(path, file):
highlighted = highlightString(fileStr)
- out = open(os.path.join(self.outputdir, file + self.newext), "w")
- out.writelines(highlighted.encode('utf-8'))
+ out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" )
+ out.write(highlighted)
out.close()
def highlightDir(path):
View
24 bin/yuidoc_parse.py
@@ -13,8 +13,9 @@
an API. It is designed to parse one module at a time '''
import os, re, simplejson, string, sys, pprint, logging, logging.config
from const import *
-from cStringIO import StringIO
+# from cStringIO import StringIO
from optparse import OptionParser
+import codecs
try:
logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -39,18 +40,21 @@ def _mkdir(newdir):
if tail: os.mkdir(newdir)
def parseFile(path, file):
- f=open(os.path.join(path, file))
+ # f=open(os.path.join(path, file))
+ f = codecs.open(os.path.join(path, file), "r", "utf-8" )
#adding a return to the beginning of the line allows for:
# #!/usr/bin/foo
# MOVED to the file marker
# fileStr = StringIO("\n%s" % f.read()).getvalue()
- fileStr = StringIO("%s" % f.read()).getvalue()
+ # fileStr = StringIO("%s" % f.read()).getvalue()
+ fileStr = f.read()
log.info("parsing " + file)
# add a file marker token so the parser can keep track of what is in what file
content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file)
# copy
- out = open(os.path.join(self.outputdir, file), "w")
+ # out = open(os.path.join(self.outputdir, file), "w")
+ out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" )
out.writelines(fileStr)
out.close()
@@ -142,9 +146,9 @@ def parseDir(path):
self.parse(self.tokenize(match))
- out = open(os.path.join(self.outputdir, outputfile), "w")
+ out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" )
- out.writelines(simplejson.dumps(self.data))
+ out.writelines(simplejson.dumps(self.data, ensure_ascii=False))
out.close()
@@ -325,15 +329,11 @@ def parseParams(tokenMap, dict, srctag=PARAM, desttag=PARAMS):
log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap)))
sys.exit()
- # description.encode('utf-8', 'xmlcharrefreplace')
- # description = unicode(description, 'utf-8', 'xmlcharrefreplace')
-
-
mo = self.param_pat.match(description)
if mo:
name = mo.group(1)
description = mo.group(2)
- description.encode('utf-8', 'xmlcharrefreplace')
+ # description.encode('utf-8', 'xmlcharrefreplace')
dict[desttag].append({
NAME: name,
@@ -436,8 +436,6 @@ def defineClass(name):
# for the block
if token and DESCRIPTION not in tokenMap:
- # token = unicode(token, 'utf-8', 'xmlcharrefreplace')
- ############################ token.encode('utf-8', 'xmlcharrefreplace')
tokenMap[DESCRIPTION] = [token]
else: pass # I don't think this can happen any longer
View
4 template/main.tmpl
@@ -1,5 +1,4 @@
#encoding UTF-8
-#filter EncodeUnicode
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#">
<head>
@@ -61,7 +60,7 @@
<style>
#doc3 .classopts { display:none; }
</style>
- #include raw $filepath_highlighted
+ $highlightcontent
</div>
#else if $classname
<h2>
@@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit();
#end if
</body>
</html>
-#end filter

0 comments on commit c438238

Please sign in to comment.