@@ -25,20 +25,6 @@ Unix (including cygwin): easy_install pygments easy_install Cheetah - - ***************************************************************************** - ***************************************************************************** - *** - *** NOTE: recent versions of Cheetah are sometimes throwing errors when - *** encountering international characters. Until this is resolved, you must - *** use Cheetah 2.1.0 or below if you are experiencing a unicode error - *** when parsing your files. - *** - *** easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west - *** - ***************************************************************************** - ***************************************************************************** - easy_install simplejson INSTALL @@ -1,14 +1,4 @@ ************************************************************************** -Unicode Errors: - -Recent versions of Cheetah are sometimes throwing errors when -encountering international characters. Until this is resolved, you must -use Cheetah 2.1.0 or below if you are experiencing unicode errors -when parsing your files. - -easy_install http://downloads.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-2.1.0.tar.gz?use_mirror=superb-west - -************************************************************************** Mapping Files to Modules: Because a single @module is supported for a set of files, the parser expects that all files in KNOWN_ISSUES @@ -12,9 +12,10 @@ version: 1.0.0b1 ''' Prints documentation with htmltmpl from the json data outputted by parser.py ''' import os, re, simplejson, shutil, logging, logging.config, time, datetime from const import * -from cStringIO import StringIO +# from cStringIO import StringIO from Cheetah.Template import Template from sets import Set +import codecs try: logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG)) @@ -68,8 +69,11 @@ class DocGenerator(object): self.showprivate = showprivate - f=open(os.path.join(inpath, datafile)) - self.rawdata = StringIO(f.read()).getvalue() + f = codecs.open(os.path.join(inpath, datafile), "r", "utf-8" ) + self.rawdata = f.read() + + # log.info('INPUT DATA: ' + self.rawdata) + d = self.data = simplejson.loads(self.rawdata) self.projectname = projectname @@ -102,11 +106,15 @@ class DocGenerator(object): # log.warn('cleansed module: %s' %(cleansed)); return self.moduleprefix + cleansed - def write(self, filename, data): - out = open(os.path.join(self.outpath, filename), "w") - out.writelines(str(data)) - # out.writelines(unicode(data)) - # out.writelines(unicode(data, 'utf-8', 'xmlcharrefreplace')) + def write(self, filename, data, template=True): + out = codecs.open( os.path.join(self.outpath, filename), "w", "utf-8" ) + + if template: + datastr = data.respond() + out.write(datastr) + else: + out.write(data) + out.close() def process(self): @@ -129,7 +137,7 @@ class DocGenerator(object): template.filename = self.filename if self.filename: template.filepath = os.path.join(self.inpath, self.filename) - template.filepath_highlighted = template.filepath + self.newext + template.highlightcontent = codecs.open(os.path.join(self.inpath, self.filename + self.newext), "r", "utf-8" ).read() template.pagetype = self.pagetype template.classmap = self.classmap @@ -267,7 +275,7 @@ class DocGenerator(object): # jsonname = self.cleansedmodulename + ".json" jsonname = "raw.json" log.info("Writing " + jsonname) - self.write(jsonname, self.rawdata) + self.write(jsonname, self.rawdata, False) for mname in self.modules: log.info("Generating module splash for %s" %(mname)) @@ -380,6 +388,7 @@ class DocGenerator(object): if self.showprivate or PRIVATE not in prop: propdata = {NAME: propertykey, HOST: i, TYPE: 'property', URL:getUrl(i, propertykey, PROPERTY)} + transferToDict( ACCESS, prop, propdata ) if PRIVATE in prop: propdata[ACCESS] = PRIVATE elif PROTECTED in prop: propdata[ACCESS] = PROTECTED @@ -398,6 +407,7 @@ class DocGenerator(object): if FINAL in prop: propdata[FINAL] = FINAL props.append(propdata) + # Methods methods = t.methods = [] if METHODS in c: @@ -574,11 +584,11 @@ class DocGenerator(object): # write module splash moduleprops.sort(allprop_sort) t.allprops_raw = moduleprops - moduleprops_json = simplejson.dumps(moduleprops) + moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False) t.allprops = moduleprops_json classList.sort(allprop_sort) t.classList_raw = classList - t.classList = simplejson.dumps(classList) + t.classList = simplejson.dumps(classList, ensure_ascii=False) self.write("%s.html" %(self.classname), t) # clear out class name @@ -594,7 +604,7 @@ class DocGenerator(object): # write module splash moduleprops.sort(allprop_sort) t.allprops_raw = moduleprops - moduleprops_json = simplejson.dumps(moduleprops) + moduleprops_json = simplejson.dumps(moduleprops, ensure_ascii=False) t.allprops = moduleprops_json # log.warn('cleansed module file name: %s' %(t.cleansedmodulename)); @@ -619,8 +629,8 @@ class DocGenerator(object): allprops.sort(allprop_sort) - allprops_json = simplejson.dumps(allprops) - self.write("index.json",allprops_json) + allprops_json = simplejson.dumps(allprops, ensure_ascii=False) + self.write("index.json", allprops_json, False) # index log.info("Generating index") @@ -666,7 +676,7 @@ class DocGenerator(object): log.warn('class map ' + i + ' failure (no module declaration?)') except: pass - t.pkgmap = simplejson.dumps(pkgMap) + t.pkgmap = simplejson.dumps(pkgMap, ensure_ascii=False) self.write("classmap.js", t) bin/yuidoc_generate.py @@ -11,7 +11,6 @@ version: 1.0.0b1 import os, re, string, logging, logging.config from const import * -from cStringIO import StringIO from optparse import OptionParser from pygments import highlight from pygments.formatters import HtmlFormatter @@ -76,8 +75,8 @@ class DocHighlighter(object): highlighted = highlightString(fileStr) - out = open(os.path.join(self.outputdir, file + self.newext), "w") - out.writelines(highlighted.encode('utf-8')) + out = codecs.open( os.path.join(self.outputdir, file + self.newext), "w", "utf-8" ) + out.write(highlighted) out.close() def highlightDir(path): bin/yuidoc_highlight.py @@ -13,8 +13,9 @@ version: 1.0.0b1 an API. It is designed to parse one module at a time ''' import os, re, simplejson, string, sys, pprint, logging, logging.config from const import * -from cStringIO import StringIO +# from cStringIO import StringIO from optparse import OptionParser +import codecs try: logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG)) @@ -39,18 +40,21 @@ class DocParser(object): if tail: os.mkdir(newdir) def parseFile(path, file): - f=open(os.path.join(path, file)) + # f=open(os.path.join(path, file)) + f = codecs.open(os.path.join(path, file), "r", "utf-8" ) #adding a return to the beginning of the line allows for: # #!/usr/bin/foo # MOVED to the file marker # fileStr = StringIO("\n%s" % f.read()).getvalue() - fileStr = StringIO("%s" % f.read()).getvalue() + # fileStr = StringIO("%s" % f.read()).getvalue() + fileStr = f.read() log.info("parsing " + file) # add a file marker token so the parser can keep track of what is in what file content = "\n/** @%s %s \n*/\n" % (FILE_MARKER, file) # copy - out = open(os.path.join(self.outputdir, file), "w") + # out = open(os.path.join(self.outputdir, file), "w") + out = codecs.open( os.path.join(self.outputdir, file), "w", "utf-8" ) out.writelines(fileStr) out.close() @@ -142,9 +146,9 @@ class DocParser(object): self.parse(self.tokenize(match)) - out = open(os.path.join(self.outputdir, outputfile), "w") + out = codecs.open( os.path.join(self.outputdir, outputfile), "w", "utf-8" ) - out.writelines(simplejson.dumps(self.data)) + out.writelines(simplejson.dumps(self.data, ensure_ascii=False)) out.close() @@ -325,15 +329,11 @@ class DocParser(object): log.error("\nError, a parameter could not be parsed:\n\n %s\n\n %s\n" %(i, pprint.pformat(tokenMap))) sys.exit() - # description.encode('utf-8', 'xmlcharrefreplace') - # description = unicode(description, 'utf-8', 'xmlcharrefreplace') - - mo = self.param_pat.match(description) if mo: name = mo.group(1) description = mo.group(2) - description.encode('utf-8', 'xmlcharrefreplace') + # description.encode('utf-8', 'xmlcharrefreplace') dict[desttag].append({ NAME: name, @@ -436,8 +436,6 @@ it was empty" % token # for the block if token and DESCRIPTION not in tokenMap: - # token = unicode(token, 'utf-8', 'xmlcharrefreplace') - ############################ token.encode('utf-8', 'xmlcharrefreplace') tokenMap[DESCRIPTION] = [token] else: pass # I don't think this can happen any longer bin/yuidoc_parse.py @@ -1,5 +1,4 @@ #encoding UTF-8 -#filter EncodeUnicode <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> <html xmlns:yui="http://yuilibrary.com/rdf/1.0/yui.rdf#"> <head> @@ -61,7 +60,7 @@ <style> #doc3 .classopts { display:none; } </style> - #include raw $filepath_highlighted + $highlightcontent </div> #else if $classname <h2> @@ -687,4 +686,3 @@ YWA.getTracker("10001393677061").submit(); #end if </body> </html> -#end filter template/main.tmpl 2affa9a64119afcf158b5e387d144cac3c5b6241 Adam Moore adamoore@yahoo-inc.com http://github.com/yui/yuidoc/commit/c4382388aed6188dffc6e4e5d67bbfb198a6cec6 c4382388aed6188dffc6e4e5d67bbfb198a6cec6 2009-07-02T18:39:14-07:00 2009-07-02T18:39:14-07:00 Completely refactored all io operations to eliminate anything encoding unicode to utf-8. Fixes unicode battles with the latest versions of Cheetah. 2bc3181880d808dd728f8d7615df16125ab3cab3 Adam Moore adamoore@yahoo-inc.com