Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Assume UTF-8 encoding for .markdown and .haml #30

Closed
wants to merge 1 commit into from
Closed

Assume UTF-8 encoding for .markdown and .haml #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 4 additions & 6 deletions lib/wheat/data.js
View file
Open in desktop
Expand Up @@ -3,12 +3,10 @@ var Git = require('git-fs'),
Step = require('step'), Step = require('step'),
util = require(process.binding('natives').util ? 'util' : 'sys'), util = require(process.binding('natives').util ? 'util' : 'sys'),
Script = process.binding('evals').Script, Script = process.binding('evals').Script,
QueryString = require('querystring'); QueryString = require('querystring'),
Tools = require('tools');


function preProcessMarkdown(markdown) { function preProcessMarkdown(markdown) {
if (!(typeof markdown === 'string')) {
markdown = markdown.toString();
}
var props = { }; var props = { };


// Parse out headers // Parse out headers
Expand Down Expand Up @@ -208,7 +206,7 @@ var Data = module.exports = {
var props; var props;
Step( Step(
function getArticleMarkdown() { function getArticleMarkdown() {
Git.readFile(version, Path.join("articles", name + ".markdown"), this); Git.readFile(version, Path.join("articles", name + ".markdown"), Tools.fixEncoding(this));
}, },
function (err, markdown) { function (err, markdown) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Expand Down Expand Up @@ -286,7 +284,7 @@ var Data = module.exports = {
callback(new Error("name is required")); callback(new Error("name is required"));
return; return;
} }
Git.readFile(version, Path.join("authors", name + ".markdown"), this); Git.readFile(version, Path.join("authors", name + ".markdown"), Tools.fixEncoding(this));
}, },
function process(err, markdown) { function process(err, markdown) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Expand Down
10 changes: 5 additions & 5 deletions lib/wheat/renderers.js
View file
Open in desktop
Expand Up @@ -108,7 +108,7 @@ var Renderers = module.exports = {
function loadData(err, head) { function loadData(err, head) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Data.articles(version, this.parallel()); Data.articles(version, this.parallel());
Git.readFile(head, "description.markdown", this.parallel()); Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel()); Data.categories(version, this.parallel());
}, },
function applyTemplate(err, articles, description, categories) { function applyTemplate(err, articles, description, categories) {
Expand Down Expand Up @@ -174,7 +174,7 @@ var Renderers = module.exports = {
if (err) { callback(err); return; } if (err) { callback(err); return; }
article = props; article = props;
insertSnippets(article.markdown, article.snippets, this.parallel()); insertSnippets(article.markdown, article.snippets, this.parallel());
Git.readFile(head, "description.markdown", this.parallel()); Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
}, },
function applyTemplate(err, markdown, description) { function applyTemplate(err, markdown, description) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Expand Down Expand Up @@ -204,7 +204,7 @@ var Renderers = module.exports = {
function loadData(err, head) { function loadData(err, head) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Data.articles(version, this.parallel()); Data.articles(version, this.parallel());
Git.readFile(head, "description.markdown", this.parallel()); Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel()); Data.categories(version, this.parallel());
}, },
function applyTemplate(err, articles, description, categories) { function applyTemplate(err, articles, description, categories) {
Expand Down Expand Up @@ -237,7 +237,7 @@ var Renderers = module.exports = {
}, },
function loadArticleFiles(err, data) { function loadArticleFiles(err, data) {
if (err) { if (err) {
Git.readFile(version, "articles/" + path, this); Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
} }
return data; return data;
}, },
Expand All @@ -262,7 +262,7 @@ var Renderers = module.exports = {
}, },
function loadArticleFiles(err, data) { function loadArticleFiles(err, data) {
if (err) { if (err) {
Git.readFile(version, "articles/" + path, this); Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
} }
return data; return data;
}, },
Expand Down
17 changes: 16 additions & 1 deletion lib/wheat/tools.js
View file
Open in desktop
Expand Up @@ -113,6 +113,20 @@ var Helpers = {


}; };


// converts 'binary' string to UTF-8 encoding
// GitreadFile always return string with no ('binary') encoding
// it would be better if Git.readFile returned a Buffer which we could convert to String
function fixEncoding(callback) {
return function(err, data) {
if(err) {
callback(err);
return;
}
data = new Buffer(data, 'binary');
callback(err, data.toString('utf8'));
};
}

// Convert UTF8 strings to binary buffers for faster loading // Convert UTF8 strings to binary buffers for faster loading
function stringToBuffer(string) { function stringToBuffer(string) {
var buffer = new Buffer(Buffer.byteLength(string)); var buffer = new Buffer(Buffer.byteLength(string));
Expand All @@ -124,7 +138,7 @@ function stringToBuffer(string) {
var loadTemplate = Git.safe(function loadTemplate(version, name, callback) { var loadTemplate = Git.safe(function loadTemplate(version, name, callback) {
Step( Step(
function loadHaml() { function loadHaml() {
Git.readFile(version, "skin/" + name + ".haml", this); Git.readFile(version, "skin/" + name + ".haml", fixEncoding(this));
}, },
function compileTemplate(err, haml) { function compileTemplate(err, haml) {
if (err) { callback(err); return; } if (err) { callback(err); return; }
Expand Down Expand Up @@ -184,6 +198,7 @@ function render(name, data, callback, partial) {
} }


module.exports = { module.exports = {
fixEncoding: fixEncoding,
stringToBuffer: stringToBuffer, stringToBuffer: stringToBuffer,
compileTemplate: compileTemplate, compileTemplate: compileTemplate,
render: render render: render
Expand Down