This repository has been archived by the owner. It is now read-only.
Permalink
Browse files

update peg-markdown sources to latest version

We've fixed some leaks in peg-markdown and added a few functions for
use in libraries. It's still not suitable for use in long-running
processes but RSS growth per-parse has decreased by about 10x.

Also, note that the peg-markdown submodule has been changed to point
to git://github.com/rtomayko/peg-markdown.git -- you may need to
remove the submodule config entry from your .git/config file and run
the submodule:init rake task.
  • Loading branch information...
rtomayko committed May 29, 2008
1 parent 8acc8ea commit 45b95de22fd779b4657bb58b72d669bc658e4db1
Showing with 566 additions and 320 deletions.
  1. +8 −0 .gitignore
  2. +1 −1 .gitmodules
  3. +44 −6 Rakefile
  4. +78 −0 ext/bufopen.c
  5. +4 −0 ext/bufopen.h
  6. +27 −71 ext/markdown.c
  7. +0 −46 ext/markdown_buffer.c
  8. +0 −6 ext/markdown_buffer.h
  9. +145 −0 ext/markdown_lib.c
  10. +18 −0 ext/markdown_lib.h
  11. +181 −162 ext/markdown_output.c
  12. +56 −14 ext/markdown_parser.c
  13. BIN ext/markdown_parser.o
  14. +3 −13 ext/markdown_peg.h
  15. +1 −1 peg-markdown
View
@@ -0,0 +1,8 @@
ext/Makefile
ext/*.o
ext/*.bundle
ext/*.so
ext/*.dll
lib/*.bundle
lib/*.so
lib/*.dll
View
@@ -1,3 +1,3 @@
[submodule "peg-markdown"]
path = peg-markdown
url = git://github.com/jgm/peg-markdown.git
url = git://github.com/rtomayko/peg-markdown.git
View
@@ -2,6 +2,8 @@ require 'rake/clean'
require 'rake/packagetask'
require 'rake/gempackagetask'
task :default => :test
DLEXT = Config::CONFIG['DLEXT']
VERS = '0.1.0'
@@ -42,19 +44,29 @@ namespace :submodule do
end
end
desc 'Update the peg-markdown submodule'
task :update => :init do
sh 'git submodule update peg-markdown'
sh 'git submodule update peg-markdown' unless File.symlink?('peg-markdown')
end
file 'peg-markdown/markdown.c' do
Rake::Task['submodule:init'].invoke
end
task :exist => 'peg-markdown/markdown.c'
end
desc 'Gather required peg-markdown sources into extension directory'
task :gather => 'submodule:update' do |t|
task :gather => 'submodule:exist' do |t|
sh 'cd peg-markdown && make markdown_parser.c'
cp FileList['peg-markdown/markdown_{peg.h,parser.c,output.c}'], 'ext/',
files =
FileList[
'peg-markdown/markdown_{peg.h,parser.c,output.c,lib.c,lib.h}',
'peg-markdown/bufopen.{c,h}'
]
cp files, 'ext/',
:preserve => true,
:verbose => true
end
CLOBBER.include 'ext/markdown_{peg.h,parser.c,output.c}'
file 'ext/Makefile' => FileList['ext/{extconf.rb,*.c,*.h,*.rb}'] do
chdir('ext') { ruby 'extconf.rb' }
@@ -79,9 +91,35 @@ task 'test:unit' => [ :build ] do |t|
end
desc 'Run conformance tests'
task 'test:conformance' => [ 'submodule:update', :build ] do |t|
task 'test:conformance' => %w[submodule:exist build] do |t|
script = "#{pwd}/bin/rpeg-markdown"
chdir('peg-markdown/MarkdownTest_1.0.3') do
sh "./MarkdownTest.pl --script=../../bin/rpeg-markdown --tidy"
sh "./MarkdownTest.pl --script='#{script}' --tidy"
end
end
desc "See how much memory we're losing"
task 'test:mem' => %w[submodule:exist build] do |t|
$: << File.join(File.dirname(__FILE__), "lib")
require 'markdown'
FileList['test.txt', 'peg-markdown/MarkdownTest_1.0.3/Tests/*.text'].each do |file|
printf "%s: \n", file
markdown = Markdown.new(File.read(file))
iterations = (ENV['N'] || 100).to_i
total, growth = [], []
iterations.times do |i|
start = Time.now
GC.start
markdown.to_html
duration = Time.now - start
GC.start
total << `ps -o rss= -p #{Process.pid}`.to_i
next if i == 0
growth << (total.last - (total[-2] || 0))
# puts "%03d: %06.02f ms / %dK used / %dK growth" % [ i, duration, total.last, growth.last ]
end
average = growth.inject(0) { |sum,x| sum + x } / growth.length
printf " %dK avg growth (per run) / %dK used (after %d runs)\n", average, total.last, iterations
end
end
View
@@ -0,0 +1,78 @@
#include "bufopen.h"
#define INCREMENT 4096
struct buffer {
char *ptr;
size_t used;
size_t len;
char **psave;
size_t *plen;
};
typedef struct buffer BUFFER ;
/* called when data is written to the stream. len is the number of bytes
* to read from data. The data buffer will not always include a terminating
* null character. */
static int buffer_write(void *cookie, const char *data, int len) {
int sz = len;
BUFFER *buf = cookie;
/* allocate a buffer capable of storing n+1 bytes. */
while ((buf->len - buf->used) <= len) {
buf->len += INCREMENT;
if ((buf->ptr = realloc(buf->ptr, buf->len)) == NULL)
return -1;
}
char *pbuf = buf->ptr + buf->used;
while (sz-- > 0)
*pbuf++ = *data++;
buf->used += len;
/* add null terminator if last char + 1 isn't already null */
if (*pbuf != '\0')
*pbuf = '\0';
return len;
}
/* called when the stream is closed */
static int buffer_close(void *cookie) {
BUFFER *buf = cookie;
*(buf->plen) = buf->len;
*(buf->psave) = buf->ptr;
free(buf);
return 0;
}
/* create a stream backed by a char buffer. most of the stream manipulation
* functions (fprintf, putc, etc.) can be used directly against the stream
* returned. The psave argument is a pointer to a char pointer that will
* receive the final buffer allocation. The plen argument is a pointer to
* a size_t that will receive the number of bytes written to the buffer.
*
* This function's interface was inspired by GNU's open_memstream. */
FILE *bufopen(size_t len, char ** psave, size_t * plen) {
FILE *rv;
BUFFER * buf;
if ((buf = malloc(sizeof *buf)) == NULL)
return NULL;
buf->len = (len == 0) ? INCREMENT : len;
buf->used = 0;
buf->ptr = malloc(buf->len);
buf->psave = psave;
buf->plen = plen;
rv = funopen(buf, NULL, buffer_write, NULL, buffer_close);
if (rv == NULL)
free(buf);
return rv;
}
// vim: ts=4 sw=4
View
@@ -0,0 +1,4 @@
#include <stdlib.h>
#include <stdio.h>
FILE *bufopen(size_t len, char ** psave, size_t * plen);
View
@@ -1,6 +1,5 @@
#include "ruby.h"
#include "markdown_peg.h"
#include "markdown_buffer.h"
#include "markdown_lib.h"
static VALUE rb_cMarkdown;
@@ -12,79 +11,36 @@ static ID id_notes;
#define INCREMENT 4096 /* size of chunks in which to allocate memory */
static VALUE
markdown_to_html(VALUE self)
rb_markdown_to_html(VALUE self)
{
element parsed_input;
VALUE output_buffer;
char *inputbuf, *curchar;
int charstotab, buflength, maxlength;
/* grab char pointer to markdown input text */
VALUE text = rb_funcall(self, id_text, 0);
Check_Type(text, T_STRING);
char * ptext = StringValuePtr(text);
buflength = 0;
maxlength = RSTRING(text)->len >= INCREMENT ?
RSTRING(text)->len :
INCREMENT;
inputbuf = malloc(maxlength);
curchar = inputbuf;
charstotab = TABSTOP;
while ((*curchar = *ptext++) != '\0') {
switch (*curchar) {
case '\t':
while (charstotab > 0)
*curchar = ' ', curchar++, buflength++, charstotab--;
break;
case '\n':
curchar++, buflength++, charstotab = TABSTOP;
break;
default:
curchar++, buflength++, charstotab--;
}
if (charstotab == 0)
charstotab = TABSTOP;
if (buflength > maxlength - TABSTOP - 3) {
maxlength += INCREMENT;
inputbuf = realloc(inputbuf, maxlength);
curchar = inputbuf + buflength;
if (inputbuf == NULL) {
/* TODO: no memory */
}
}
}
*curchar++ = '\n';
*curchar++ = '\n';
*curchar = '\0';
buflength+= 2;
/* flip extension bits */
int extensions = 0;
if ( rb_funcall(self, id_smart, 0) == Qtrue )
extensions = extensions | EXT_SMART ;
if ( rb_funcall(self, id_notes, 0) == Qtrue )
extensions = extensions | EXT_NOTES ;
/* parse markdown input into sematic element tree */
parsed_input = markdown(inputbuf, extensions);
/* allocate output buffer and generate output */
output_buffer = rb_markdown_buffer_init(buflength * 2);
print_element(parsed_input, HTML_FORMAT);
rb_markdown_buffer_free();
return output_buffer;
/* grab char pointer to markdown input text */
VALUE text = rb_funcall(self, id_text, 0);
Check_Type(text, T_STRING);
char * ptext = StringValuePtr(text);
/* flip extension bits */
int extensions = 0;
if ( rb_funcall(self, id_smart, 0) == Qtrue )
extensions = extensions | EXT_SMART ;
if ( rb_funcall(self, id_notes, 0) == Qtrue )
extensions = extensions | EXT_NOTES ;
char *html = markdown_to_string(ptext, extensions, HTML_FORMAT);
VALUE result = rb_str_new2(html);
free(html);
return result;
}
void Init_markdown()
{
/* Initialize frequently used Symbols */
id_text = rb_intern("text");
id_smart = rb_intern("smart");
id_notes = rb_intern("notes");
/* Initialize frequently used Symbols */
id_text = rb_intern("text");
id_smart = rb_intern("smart");
id_notes = rb_intern("notes");
rb_cMarkdown = rb_define_class("Markdown", rb_cObject);
rb_define_method(rb_cMarkdown, "to_html", markdown_to_html, 0);
rb_cMarkdown = rb_define_class("Markdown", rb_cObject);
rb_define_method(rb_cMarkdown, "to_html", rb_markdown_to_html, 0);
}
// vim: ts=4 sw=4
View

This file was deleted.

Oops, something went wrong.
View

This file was deleted.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit 45b95de

Please sign in to comment.