Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Assume UTF-8 encoding for .markdown and .haml #30

Closed
wants to merge 1 commit into from

2 participants

@pirxpilot

Alternative fix for #29 and #16

New Tools.fixEncoding function can be used to adjust the result of
Git.readFile whenever we expect UTF-8 encoded file.

Should be pretty safe since ASCII is UFT-8
The real culprit here is git-fs which reads files to string using 'binary' encoding
We convert the file content back to buffer and re-encode it as UTF-8

It would be better if git-fs were fixed to read files content into buffer (and not into
string as it does now). But even after git-fs is fixed this code should work.

@pirxpilot pirxpilot Assume UTF-8 encoding for .markdown and .haml
New Tools.fixEncoding function can be used to adjust the result of
Git.readFile whenever we expect UTF-8 encoded file.

Should be pretty safe since ASCII is UFT-8
The real culprit here is git-fs which reads files to string using 'binary' encoding
We convert the file content back to buffer and re-encode it as UTF-8

It would be better if git-fs were fixed to read files content into buffer (and not into
string as it does now).
db4ef96
@TooTallNate

Considering that the binary encoding has been deprecated for some time now, I think the real fix would be to make git-fs return Buffers, instead of binary Strings. Then UTF8 support would be as simple as buf.toString('utf8')...

@pirxpilot

That's kind of what I was trying to say ;-)
I don't know how many people/apps depend on Git.readFile returning strings. I'll try to prepare git-fs patch as well, but - as I said in the comment - even after git-fs is patched this code will still work, since the conversion to 'binary' only kicks in if Git.read file returns string.

@pirxpilot

This is node-git pull request for fixing Git.fileRead

@pirxpilot

Looks like @creationix pulled my git-fs patch. I am closing this one and will publish a patch based on new git-fs functionality.

@pirxpilot pirxpilot closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Aug 25, 2011
  1. @pirxpilot

    Assume UTF-8 encoding for .markdown and .haml

    pirxpilot authored
    New Tools.fixEncoding function can be used to adjust the result of
    Git.readFile whenever we expect UTF-8 encoded file.
    
    Should be pretty safe since ASCII is UFT-8
    The real culprit here is git-fs which reads files to string using 'binary' encoding
    We convert the file content back to buffer and re-encode it as UTF-8
    
    It would be better if git-fs were fixed to read files content into buffer (and not into
    string as it does now).
This page is out of date. Refresh to see the latest.
Showing with 25 additions and 12 deletions.
  1. +4 −6 lib/wheat/data.js
  2. +5 −5 lib/wheat/renderers.js
  3. +16 −1 lib/wheat/tools.js
View
10 lib/wheat/data.js
@@ -3,12 +3,10 @@ var Git = require('git-fs'),
Step = require('step'),
util = require(process.binding('natives').util ? 'util' : 'sys'),
Script = process.binding('evals').Script,
- QueryString = require('querystring');
+ QueryString = require('querystring'),
+ Tools = require('tools');
function preProcessMarkdown(markdown) {
- if (!(typeof markdown === 'string')) {
- markdown = markdown.toString();
- }
var props = { };
// Parse out headers
@@ -208,7 +206,7 @@ var Data = module.exports = {
var props;
Step(
function getArticleMarkdown() {
- Git.readFile(version, Path.join("articles", name + ".markdown"), this);
+ Git.readFile(version, Path.join("articles", name + ".markdown"), Tools.fixEncoding(this));
},
function (err, markdown) {
if (err) { callback(err); return; }
@@ -286,7 +284,7 @@ var Data = module.exports = {
callback(new Error("name is required"));
return;
}
- Git.readFile(version, Path.join("authors", name + ".markdown"), this);
+ Git.readFile(version, Path.join("authors", name + ".markdown"), Tools.fixEncoding(this));
},
function process(err, markdown) {
if (err) { callback(err); return; }
View
10 lib/wheat/renderers.js
@@ -108,7 +108,7 @@ var Renderers = module.exports = {
function loadData(err, head) {
if (err) { callback(err); return; }
Data.articles(version, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel());
},
function applyTemplate(err, articles, description, categories) {
@@ -174,7 +174,7 @@ var Renderers = module.exports = {
if (err) { callback(err); return; }
article = props;
insertSnippets(article.markdown, article.snippets, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
},
function applyTemplate(err, markdown, description) {
if (err) { callback(err); return; }
@@ -204,7 +204,7 @@ var Renderers = module.exports = {
function loadData(err, head) {
if (err) { callback(err); return; }
Data.articles(version, this.parallel());
- Git.readFile(head, "description.markdown", this.parallel());
+ Git.readFile(head, "description.markdown", Tools.fixEncoding(this.parallel()));
Data.categories(version, this.parallel());
},
function applyTemplate(err, articles, description, categories) {
@@ -237,7 +237,7 @@ var Renderers = module.exports = {
},
function loadArticleFiles(err, data) {
if (err) {
- Git.readFile(version, "articles/" + path, this);
+ Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
}
return data;
},
@@ -262,7 +262,7 @@ var Renderers = module.exports = {
},
function loadArticleFiles(err, data) {
if (err) {
- Git.readFile(version, "articles/" + path, this);
+ Git.readFile(version, "articles/" + path, Tools.fixEncoding(this));
}
return data;
},
View
17 lib/wheat/tools.js
@@ -113,6 +113,20 @@ var Helpers = {
};
+// converts 'binary' string to UTF-8 encoding
+// GitreadFile always return string with no ('binary') encoding
+// it would be better if Git.readFile returned a Buffer which we could convert to String
+function fixEncoding(callback) {
+ return function(err, data) {
+ if(err) {
+ callback(err);
+ return;
+ }
+ data = new Buffer(data, 'binary');
+ callback(err, data.toString('utf8'));
+ };
+}
+
// Convert UTF8 strings to binary buffers for faster loading
function stringToBuffer(string) {
var buffer = new Buffer(Buffer.byteLength(string));
@@ -124,7 +138,7 @@ function stringToBuffer(string) {
var loadTemplate = Git.safe(function loadTemplate(version, name, callback) {
Step(
function loadHaml() {
- Git.readFile(version, "skin/" + name + ".haml", this);
+ Git.readFile(version, "skin/" + name + ".haml", fixEncoding(this));
},
function compileTemplate(err, haml) {
if (err) { callback(err); return; }
@@ -184,6 +198,7 @@ function render(name, data, callback, partial) {
}
module.exports = {
+ fixEncoding: fixEncoding,
stringToBuffer: stringToBuffer,
compileTemplate: compileTemplate,
render: render
Something went wrong with that request. Please try again.