Fixed utf related highlighter crash [fixes yui#22]

bin/yuidoc_highlight.py

@@ -17,6 +17,7 @@
 from pygments.lexers import JavascriptLexer
 from pygments.lexers import PhpLexer
 from pygments.formatters import HtmlFormatter
+import codecs
 
 try:
     logging.config.fileConfig(os.path.join(sys.path[0], LOGCONFIG))
@@ -51,7 +52,8 @@ def highlightString(src):
 
         def highlightFile(path, file):
             f=open(os.path.join(path, file))
-            fileStr=StringIO(f.read()).getvalue()
+            fileStr = codecs.open( os.path.join(path, file), "r", "utf-8" ).read()
+
             f.close()
             log.info("highlighting " + file)
 

test/dd.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+##############################################################################
+
+# The location of your yuidoc install
+# yuidoc_home=yahoo/presentation/tools/yuidoc
+# yuidoc_home=~/www/yuidoc/yuidoc
+yuidoc_home=..
+
+src=~/src/yui3/src
+
+parser_in="$src/dd/js"
+
+# The location to output the parser data.  This output is a file containing a 
+# json string, and copies of the parsed files.
+parser_out=build_tmp/yuidoc_tmp
+
+# The directory to put the html file outputted by the generator
+generator_out=build_tmp/api
+
+# The location of the template files.  Any subdirectories here will be copied
+# verbatim to the destination directory.
+template=$yuidoc_home/template
+
+version="localtest"
+
+##############################################################################
+
+$yuidoc_home/bin/yuidoc.py $parser_in -p $parser_out -o $generator_out -t $template -v $version -s $*
+

test/intl.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+##############################################################################
+
+# The location of your yuidoc install
+# yuidoc_home=yahoo/presentation/tools/yuidoc
+# yuidoc_home=~/www/yuidoc/yuidoc
+yuidoc_home=..
+
+src=.
+
+parser_in="$src/intl"
+
+# The location to output the parser data.  This output is a file containing a 
+# json string, and copies of the parsed files.
+parser_out=build_tmp/yuidoc_tmp
+
+# The directory to put the html file outputted by the generator
+generator_out=build_tmp/api
+
+# The location of the template files.  Any subdirectories here will be copied
+# verbatim to the destination directory.
+template=$yuidoc_home/template
+
+version="localtest"
+
+##############################################################################
+
+$yuidoc_home/bin/yuidoc.py $parser_in -p $parser_out -o $generator_out -t $template -v $version -s $*
+

test/intl/intl.js

@@ -0,0 +1,33 @@
+/**
+ * The text module lets you format and parse text in many languages from around the world.
+ * <p>This module uses the following identifiers:
+ * <ul>
+ * <li><em>Languages:</em> RFC 4646 language tags, such as "en-GB" (English as used
+ *     in the United Kingdom), "zh-Hans-CN" (simpified Chinese as used in China).
+ * <li><em>Time zones:</em> tz database identifiers, such as "Europe/Berlin"
+ *     (time zone of Germany), "America/Los_Angeles" (Pacific time zone in the United States),
+ *     "Asia/Kolkata" (time zone of India).
+ * </ul>
+ * @module text
+ * @requires yahoo, json
+ */
+
+/** 
+ * Provides functionality for formatting date and time information.
+ * @class DateFormat
+ * @namespace YAHOO.text
+ * @constructor
+ * @param {Style | Style[] | String} style Style constant or pattern string for the desired date format.
+ * @param {String} language The RFC 4646 language tag for the language of the date format.
+ * @param {String} timeZone <b>future</b> The tz database identifier for the time zone of the date format.
+ *     Optional - the browser time zone is used by default.
+ */
+
+    /**
+     * <b>future</b> Calendar month format. Examples:
+     * <ul>
+     * <li>June 2009
+     * <li>2009年6月
+     * </ul>
+     * @property CALENDAR_MONTH
+     */

test/json.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+##############################################################################
+
+# The location of your yuidoc install
+# yuidoc_home=yahoo/presentation/tools/yuidoc
+# yuidoc_home=~/www/yuidoc/yuidoc
+yuidoc_home=..
+
+src=~/src/yui3/src
+
+parser_in="$src/io/js $src/json/js"
+
+# The location to output the parser data.  This output is a file containing a 
+# json string, and copies of the parsed files.
+parser_out=build_tmp/yuidoc_tmp
+
+# The directory to put the html file outputted by the generator
+generator_out=build_tmp/api
+
+# The location of the template files.  Any subdirectories here will be copied
+# verbatim to the destination directory.
+template=$yuidoc_home/template
+
+version=test
+
+##############################################################################
+
+$yuidoc_home/bin/yuidoc.py $parser_in -p $parser_out -o $generator_out -t $template -v $version -s $*
+