Skip to content

Commit

Permalink
relax string length limitation to 64KB; fix #2725
Browse files Browse the repository at this point in the history
  • Loading branch information
matz committed Jul 12, 2016
1 parent 2e74a93 commit d4d807b
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 19 deletions.
9 changes: 6 additions & 3 deletions include/mruby/compile.h
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ struct mrb_parser_heredoc_info {
mrb_ast_node *doc;
};

#define MRB_PARSER_BUF_SIZE 1024
#define MRB_PARSER_TOKBUF_MAX 65536
#define MRB_PARSER_TOKBUF_SIZE 256

/* parser structure */
struct mrb_parser_state {
Expand Down Expand Up @@ -130,8 +131,10 @@ struct mrb_parser_state {
mrb_ast_node *locals;

mrb_ast_node *pb;
char buf[MRB_PARSER_BUF_SIZE];
int bidx;
char *tokbuf;
char buf[MRB_PARSER_TOKBUF_SIZE];
int tidx;
int tsiz;

mrb_ast_node *all_heredocs; /* list of mrb_parser_heredoc_info* */
mrb_ast_node *heredocs_from_nextline;
Expand Down
51 changes: 35 additions & 16 deletions mrbgems/mruby-compiler/core/parse.y
Original file line number Diff line number Diff line change
Expand Up @@ -3573,15 +3573,20 @@ skips(parser_state *p, const char *s)
static int
newtok(parser_state *p)
{
p->bidx = 0;
if (p->tokbuf != p->buf) {
mrb_free(p->mrb, p->tokbuf);
p->tokbuf = p->buf;
p->tsiz = MRB_PARSER_TOKBUF_SIZE;
}
p->tidx = 0;
return p->column - 1;
}

static void
tokadd(parser_state *p, int32_t c)
{
char utf8[4];
unsigned len;
int i, len;

/* mrb_assert(-0x10FFFF <= c && c <= 0xFF); */
if (c >= 0) {
Expand Down Expand Up @@ -3615,42 +3620,51 @@ tokadd(parser_state *p, int32_t c)
len = 4;
}
}
if (p->bidx+len <= MRB_PARSER_BUF_SIZE) {
unsigned i;
for (i = 0; i < len; i++) {
p->buf[p->bidx++] = utf8[i];
if (p->tidx+len >= p->tsiz) {
if (p->tsiz >= MRB_PARSER_TOKBUF_MAX) {
p->tidx += len;
return;
}
p->tsiz *= 2;
if (p->tokbuf == p->buf) {
p->tokbuf = (char*)mrb_malloc(p->mrb, p->tsiz);
memcpy(p->tokbuf, p->buf, MRB_PARSER_TOKBUF_SIZE);
}
else {
p->tokbuf = (char*)mrb_realloc(p->mrb, p->tokbuf, p->tsiz);
}
}
for (i = 0; i < len; i++) {
p->tokbuf[p->tidx++] = utf8[i];
}
}

static int
toklast(parser_state *p)
{
return p->buf[p->bidx-1];
return p->tokbuf[p->tidx-1];
}

static void
tokfix(parser_state *p)
{
int i = p->bidx, imax = MRB_PARSER_BUF_SIZE - 1;

if (i > imax) {
i = imax;
if (p->tidx >= MRB_PARSER_TOKBUF_MAX) {
p->tidx = MRB_PARSER_TOKBUF_MAX-1;
yyerror(p, "string too long (truncated)");
}
p->buf[i] = '\0';
p->tokbuf[p->tidx] = '\0';
}

static const char*
tok(parser_state *p)
{
return p->buf;
return p->tokbuf;
}

static int
toklen(parser_state *p)
{
return p->bidx;
return p->tidx;
}

#define IS_ARG() (p->lstate == EXPR_ARG || p->lstate == EXPR_CMDARG)
Expand Down Expand Up @@ -5196,7 +5210,7 @@ parser_yylex(parser_state *p)
c = nextc(p);
}
if (c < 0) {
if (p->bidx == 1) {
if (p->tidx == 1) {
yyerror(p, "incomplete instance variable syntax");
}
else {
Expand All @@ -5205,7 +5219,7 @@ parser_yylex(parser_state *p)
return 0;
}
else if (isdigit(c)) {
if (p->bidx == 1) {
if (p->tidx == 1) {
yyerror_i(p, "'@%c' is not allowed as an instance variable name", c);
}
else {
Expand Down Expand Up @@ -5486,6 +5500,8 @@ mrb_parser_new(mrb_state *mrb)
#if defined(PARSER_TEST) || defined(PARSER_DEBUG)
yydebug = 1;
#endif
p->tsiz = MRB_PARSER_TOKBUF_SIZE;
p->tokbuf = p->buf;

p->lex_strterm = NULL;
p->all_heredocs = p->parsing_heredoc = NULL;
Expand All @@ -5501,6 +5517,9 @@ mrb_parser_new(mrb_state *mrb)
MRB_API void
mrb_parser_free(parser_state *p) {
mrb_pool_close(p->pool);
if (p->tokbuf != p->buf) {
mrb_free(p->mrb, p->tokbuf);
}
}

MRB_API mrbc_context*
Expand Down

0 comments on commit d4d807b

Please sign in to comment.