Browse files

Assume UTF-8 encoding for .markdown and .haml

New Tools.fixEncoding function can be used to adjust the result of
Git.readFile whenever we expect UTF-8 encoded file.

Should be pretty safe since ASCII is UFT-8
The real culprit here is git-fs which reads files to string using 'binary' encoding
We convert the file content back to buffer and re-encode it as UTF-8

It would be better if git-fs were fixed to read files content into buffer (and not into
string as it does now).
  • Loading branch information...
1 parent 946deaa commit db4ef966696355c71327fc143c07953063049bc9 @pirxpilot committed Aug 25, 2011
Showing with 25 additions and 12 deletions.
  1. +4 −6 lib/wheat/data.js
  2. +5 −5 lib/wheat/renderers.js
  3. +16 −1 lib/wheat/tools.js
View
10 lib/wheat/data.js
@@ -3,12 +3,10 @@ var Git = require('git-fs'),
Step = require('step'),
util = require(process.binding('natives').util ? 'util' : 'sys'),
Script = process.binding('evals').Script,
- QueryString = require('querystring');
+ QueryString = require('querystring'),
+ Tools = require('tools');
function preProcessMarkdown(markdown) {
- if (!(typeof markdown === 'string')) {
- markdown = markdown.toString();
- }
var props = { };
// Parse out headers
@@ -208,7 +206,7 @@ var Data = module.exports = {
var props;
Step(
function getArticleMarkdown() {
- Git.readFile(version, Path.join("articles", name + ".markdown"), this);
+ Git.readFile(version, Path.join("articles", name + ".markdown"), Tools.fixEncoding(this));
},
function (err, markdown) {
if (err) { callback(err); return; }
@@ -286,7 +284,7 @@ var Data = module.exports = {
callback(new Error("name is required"));
return;
}
- Git.readFile(version, Path.join("authors", name + ".markdown"), this);
+ Git.readFile(version, Path.join("authors", name + ".markdown"), Tools.fixEncoding(this));
},
function process(err, markdown) {
if (err) { callback(err); return; }
View
10 lib/wheat/renderers.js
@@ -108,7 +108,7 @@ var Renderers = module.exports = {
function loadData(err, head) {
if (err) { callback(err); return; }
Data.articles(version, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel());
},
function applyTemplate(err, articles, description, categories) {
@@ -174,7 +174,7 @@ var Renderers = module.exports = {
if (err) { callback(err); return; }
article = props;
insertSnippets(article.markdown, article.snippets, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
},
function applyTemplate(err, markdown, description) {
if (err) { callback(err); return; }
@@ -204,7 +204,7 @@ var Renderers = module.exports = {
function loadData(err, head) {
if (err) { callback(err); return; }
Data.articles(version, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel());
},
function applyTemplate(err, articles, description, categories) {
@@ -237,7 +237,7 @@ var Renderers = module.exports = {
},
function loadArticleFiles(err, data) {
if (err) {
- Git.readFile(version, "articles/" + path, this);
+ Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
}
return data;
},
@@ -262,7 +262,7 @@ var Renderers = module.exports = {
},
function loadArticleFiles(err, data) {
if (err) {
- Git.readFile(version, "articles/" + path, this);
+ Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
}
return data;
},
View
17 lib/wheat/tools.js
@@ -113,6 +113,20 @@ var Helpers = {
};
+// converts 'binary' string to UTF-8 encoding
+// GitreadFile always return string with no ('binary') encoding
+// it would be better if Git.readFile returned a Buffer which we could convert to String
+function fixEncoding(callback) {
+ return function(err, data) {
+ if(err) {
+ callback(err);
+ return;
+ }
+ data = new Buffer(data, 'binary');
+ callback(err, data.toString('utf8'));
+ };
+}
+
// Convert UTF8 strings to binary buffers for faster loading
function stringToBuffer(string) {
var buffer = new Buffer(Buffer.byteLength(string));
@@ -124,7 +138,7 @@ function stringToBuffer(string) {
var loadTemplate = Git.safe(function loadTemplate(version, name, callback) {
Step(
function loadHaml() {
- Git.readFile(version, "skin/" + name + ".haml", this);
+ Git.readFile(version, "skin/" + name + ".haml", fixEncoding(this));
},
function compileTemplate(err, haml) {
if (err) { callback(err); return; }
@@ -184,6 +198,7 @@ function render(name, data, callback, partial) {
}
module.exports = {
+ fixEncoding: fixEncoding,
stringToBuffer: stringToBuffer,
compileTemplate: compileTemplate,
render: render

0 comments on commit db4ef96

Please sign in to comment.