Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Completely refactored all io operations to eliminate anything encodin…

…g unicode to utf-8. Fixes unicode battles with the latest versions of Cheetah.
  • Loading branch information...
commit c4382388aed6188dffc6e4e5d67bbfb198a6cec6 1 parent 2affa9a
Adam Moore authored
View
14 INSTALL
@@ -25,20 +25,6 @@ Unix (including cygwin):
easy_install pygments
easy_install Cheetah
-
- *****************************************************************************
- *****************************************************************************
- ***
- *** NOTE: recent versions of Cheetah are sometimes throwing errors when
- *** encountering international characters. Until this is resolved, you must
- *** use Cheetah 2.1.0 or below if you are experiencing a unicode error
- *** when parsing your files.
- ***
- *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
- ***
- *****************************************************************************
- *****************************************************************************
-
easy_install simplejson
View
10 KNOWN_ISSUES
@@ -1,14 +1,4 @@
**************************************************************************
-Unicode Errors:
-
-Recent versions of Cheetah are sometimes throwing errors when
-encountering international characters. Until this is resolved, you must
-use Cheetah 2.1.0 or below if you are experiencing unicode errors
-when parsing your files.
-
-easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west
-
-**************************************************************************
Mapping Files to Modules:
Because a single @module is supported for a set of files, the parser expects that all files in
View
42 bin/yuidoc_generate.py
@@ -12,9 +12,10 @@
''' Prints documentation with htmltmpl from the json data outputted by parser.py '''
import os, re, simplejson, shutil, logging, logging.config, time, datetime
from const import *
-from cStringIO import StringIO
+# from cStringIO import StringIO
from Cheetah.Template import Template
from sets import Set
+import codecs
try:
logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -68,8 +69,11 @@ def _mkdir(newdir):
self.showprivate = showprivate
- f=open(os.path.join(inpath, datafile))
- self.rawdata = StringIO(f.read()).getvalue()
+ f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" )
+ self.rawdata = f.read()
+
+ # log.info('INPUT DATA: ' + self.rawdata)
+
d = self.data = simplejson.loads(self.rawdata)
self.projectname = projectname
@@ -102,11 +106,15 @@ def cleanseStr(self, strg):
# log.warn('cleansed module: %s' %(cleansed));
return self.moduleprefix + cleansed
- def write(self, filename, data):
- out = open(os.path.join(self.outpath, filename), "w")
- out.writelines(str(data))
- # out.writelines(unicode(data))
- # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace'))
+ def write(self, filename, data, template=True):
+ out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" )
+
+ if template:
+ datastr = data.respond()
+ out.write(datastr)
+ else:
+ out.write(data)
+
out.close()
def process(self):
@@ -129,7 +137,7 @@ def assignGlobalProperties(template):
template.filename = self.filename
if self.filename:
template.filepath = os.path.join(self.inpath, self.filename)
- template.filepath_highlighted = template.filepath + self.newext
+ template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read()
template.pagetype = self.pagetype
template.classmap = self.classmap
@@ -267,7 +275,7 @@ def allprop_sort(x, y):
# jsonname = self.cleansedmodulename + ".json"
jsonname = "raw.json"
log.info("Writing " + jsonname)
- self.write(jsonname, self.rawdata)
+ self.write(jsonname, self.rawdata, False)
for mname in self.modules:
log.info("Generating module splash for %s" %(mname))
@@ -380,6 +388,7 @@ def allprop_sort(x, y):
if self.showprivate or PRIVATE not in prop:
propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)}
+
transferToDict( ACCESS, prop, propdata )
if PRIVATE in prop: propdata[ACCESS] = PRIVATE
elif PROTECTED in prop: propdata[ACCESS] = PROTECTED
@@ -398,6 +407,7 @@ def allprop_sort(x, y):
if FINAL in prop: propdata[FINAL] = FINAL
props.append(propdata)
+
# Methods
methods = t.methods = []
if METHODS in c:
@@ -574,11 +584,11 @@ def allprop_sort(x, y):
# write module splash
moduleprops.sort(allprop_sort)
t.allprops_raw = moduleprops
- moduleprops_json = simplejson.dumps(moduleprops)
+ moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False)
t.allprops = moduleprops_json
classList.sort(allprop_sort)
t.classList_raw = classList
- t.classList = simplejson.dumps(classList)
+ t.classList = simplejson.dumps(classList, ensure_ascii=False)
self.write("%s.html" %(self.classname), t)
# clear out class name
@@ -594,7 +604,7 @@ def allprop_sort(x, y):
# write module splash
moduleprops.sort(allprop_sort)
t.allprops_raw = moduleprops
- moduleprops_json = simplejson.dumps(moduleprops)
+ moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False)
t.allprops = moduleprops_json
# log.warn('cleansed module file name: %s' %(t.cleansedmodulename));
@@ -619,8 +629,8 @@ def allprop_sort(x, y):
allprops.sort(allprop_sort)
- allprops_json = simplejson.dumps(allprops)
- self.write("index.json",allprops_json)
+ allprops_json = simplejson.dumps(allprops, ensure_ascii=False)
+ self.write("index.json", allprops_json, False)
# index
log.info("Generating index")
@@ -666,7 +676,7 @@ def allprop_sort(x, y):
log.warn('class map ' + i + ' failure (no module declaration?)')
except: pass
- t.pkgmap = simplejson.dumps(pkgMap)
+ t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False)
self.write("classmap.js", t)
View
5 bin/yuidoc_highlight.py
@@ -11,7 +11,6 @@
import os, re, string, logging, logging.config
from const import *
-from cStringIO import StringIO
from optparse import OptionParser
from pygments import highlight
from pygments.formatters import HtmlFormatter
@@ -76,8 +75,8 @@ def highlightFile(path, file):
highlighted = highlightString(fileStr)
- out = open(os.path.join(self.outputdir, file + self.newext), "w")
- out.writelines(highlighted.encode('utf-8'))
+ out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" )
+ out.write(highlighted)
out.close()
def highlightDir(path):
View
24 bin/yuidoc_parse.py
@@ -13,8 +13,9 @@
an API. It is designed to parse one module at a time '''
import os, re, simplejson, string, sys, pprint, logging, logging.config
from const import *
-from cStringIO import StringIO
+# from cStringIO import StringIO
from optparse import OptionParser
+import codecs
try:
logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -39,18 +40,21 @@ def _mkdir(newdir):
if tail: os.mkdir(newdir)
def parseFile(path, file):
- f=open(os.path.join(path, file))
+ # f=open(os.path.join(path, file))
+ f = codecs.open(os.path.join(path, file), "r", "utf-8" )
#adding a return to the beginning of the line allows for:
# #!/usr/bin/foo
# MOVED to the file marker
# fileStr = StringIO("\n%s" % f.read()).getvalue()
- fileStr = StringIO("%s" % f.read()).getvalue()
+ # fileStr = StringIO("%s" % f.read()).getvalue()
+ fileStr = f.read()
log.info("parsing " + file)
# add a file marker token so the parser can keep track of what is in what file
content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file)
# copy
- out = open(os.path.join(self.outputdir, file), "w")
+ # out = open(os.path.join(self.outputdir, file), "w")
+ out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" )
out.writelines(fileStr)
out.close()
@@ -142,9 +146,9 @@ def parseDir(path):
self.parse(self.tokenize(match))
- out = open(os.path.join(self.outputdir, outputfile), "w")
+ out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" )
- out.writelines(simplejson.dumps(self.data))
+ out.writelines(simplejson.dumps(self.data, ensure_ascii=False))
out.close()
@@ -325,15 +329,11 @@ def parseParams(tokenMap, dict, srctag=PARAM, desttag=PARAMS):
log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap)))
sys.exit()
- # description.encode('utf-8', 'xmlcharrefreplace')
- # description = unicode(description, 'utf-8', 'xmlcharrefreplace')
-
-
mo = self.param_pat.match(description)
if mo:
name = mo.group(1)
description = mo.group(2)
- description.encode('utf-8', 'xmlcharrefreplace')
+ # description.encode('utf-8', 'xmlcharrefreplace')
dict[desttag].append({
NAME: name,
@@ -436,8 +436,6 @@ def defineClass(name):
# for the block
if token and DESCRIPTION not in tokenMap:
- # token = unicode(token, 'utf-8', 'xmlcharrefreplace')
- ############################ token.encode('utf-8', 'xmlcharrefreplace')
tokenMap[DESCRIPTION] = [token]
else: pass # I don't think this can happen any longer
View
4 template/main.tmpl
@@ -1,5 +1,4 @@
#encoding UTF-8
-#filter EncodeUnicode
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#">
<head>
@@ -61,7 +60,7 @@
<style>
#doc3 .classopts { display:none; }
</style>
- #include raw $filepath_highlighted
+ $highlightcontent
</div>
#else if $classname
<h2>
@@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit();
#end if
</body>
</html>
-#end filter
Please sign in to comment.
Something went wrong with that request. Please try again.