Skip to content

Commit

Permalink
Merge pull request #45 from qorelanguage/bugfix/2343_single_quoted_st…
Browse files Browse the repository at this point in the history 
…ring_fix

Bugfix/2343 single quoted string fix
  • Loading branch information
Ondrej Musil committed Oct 26, 2017
2 parents 5cdf76e + f337064 commit 61e167b
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 20 deletions.
2 changes: 2 additions & 0 deletions docs/mainpage.doxygen.tmpl
View file
Expand Up @@ -96,6 +96,8 @@ any data = parse_yaml(yaml_str);
- added the \c YamlRpcConnection class to the <a href="../../YamlRpcClient/html/index.html">YamlRpcClient</a> module
- updated the <a href="../../DataStreamClient/html/index.html">DataStreamClient</a> module for complex types and new internal RestClient API changes (<a href="https://github.com/qorelanguage/qore/issues/2073">issue 2073</a>)
- updated the <a href="../../DataStreamUtil/html/index.html">DataStreamUtil</a> module for complex types
- fixed a bug deserializing single-quoted strings; also serialized \c "!number" values will always include the tag to avoid
potential future ambiguity (<a href="https://github.com/qorelanguage/qore/issues/2343">issue 2343</a>)

@subsection yaml051 yaml Module Version 0.5.1

Expand Down
1 change: 1 addition & 0 deletions src/QoreYamlEmitter.cpp
View file
Expand Up @@ -195,5 +195,6 @@ int QoreYamlEmitter::emitValue(const DateTime &d) {
else
d.format(str, "Z");

// issue #2343: to avoid ambiguity with single quoted strings, we always use the tag here
return emitScalar(str, YAML_TIMESTAMP_TAG);
}
44 changes: 32 additions & 12 deletions src/QoreYamlParser.cpp
View file
Expand Up @@ -213,7 +213,8 @@ static QoreNumberNode* parseNumber(const char* val, size_t len) {
}

// FIXME: this is still capable of false positives
static AbstractQoreNode* try_parse_number(const char* val, size_t len) {
static AbstractQoreNode* try_parse_number(const char* val, size_t len, bool no_simple_numeric = false) {
//printd(5, "try_parse_number() val: \"%s\" len: %d\n", val, (int)len);
bool sign = (*val == '-' || *val == '+');

// check for @inf@ and @nan@
Expand All @@ -235,7 +236,7 @@ static AbstractQoreNode* try_parse_number(const char* val, size_t len) {
return new QoreNumberNode(val, prec);
}
}
return 0;
return nullptr;
}

const char* str = val + (int)sign;
Expand Down Expand Up @@ -266,21 +267,21 @@ static AbstractQoreNode* try_parse_number(const char* val, size_t len) {
od = false;
if (*str == '.') {
if (dp || e || pm)
return 0;
return nullptr;
dp = true;
}
else if (*str == 'e' || *str == 'E') {
if (e || pm)
return 0;
return nullptr;
e = true;
}
else if (*str == '+' || *str == '-') {
if (pm || !e)
return 0;
return nullptr;
pm = true;
}
else
return 0;
return nullptr;
++str;
}

Expand All @@ -293,6 +294,8 @@ static AbstractQoreNode* try_parse_number(const char* val, size_t len) {
|| (len == 20 && sign
&& ((*val == '+' && strcmp(val, "+9223372036854775807") <= 0)
|| (*val == '-' && strcmp(val, "-9223372036854775808") <= 0))))) {
if (no_simple_numeric)
return nullptr;
int64 iv = strtoll(val, 0, 10);
errno = 0;
assert(errno != ERANGE);
Expand All @@ -304,7 +307,7 @@ static AbstractQoreNode* try_parse_number(const char* val, size_t len) {
return new QoreNumberNode(val);
}

return new QoreFloatNode(strtod(val, 0));
return no_simple_numeric ? nullptr : new QoreFloatNode(strtod(val, 0));
}

AbstractQoreNode* QoreYamlParser::parseScalar(bool favor_string) {
Expand All @@ -316,11 +319,23 @@ AbstractQoreNode* QoreYamlParser::parseScalar(bool favor_string) {
//printd(5, "QoreYamlParser::parseScalar() anchor=%s tag=%s value=%s len=%d plain_implicit=%d quoted_implicit=%d style=%d\n", event.data.scalar.anchor ? event.data.scalar.anchor : (yaml_char_t*)"n/a", event.data.scalar.tag ? event.data.scalar.tag : (yaml_char_t*)"n/a", val, len, event.data.scalar.plain_implicit, event.data.scalar.quoted_implicit, event.data.scalar.style);

if (!event.data.scalar.tag) {
if (favor_string || (event.data.scalar.quoted_implicit && event.data.scalar.style == YAML_DOUBLE_QUOTED_SCALAR_STYLE)) {
if (favor_string || (event.data.scalar.quoted_implicit
&& (event.data.scalar.style == YAML_DOUBLE_QUOTED_SCALAR_STYLE))) {
// assume it's a string
return new QoreStringNode(val, len, QCS_UTF8);
}

// issue #2343: could be a string, arbitrary-precision numeric or special floating-point value, or an ISO-8601 date/time value
if (event.data.scalar.quoted_implicit && event.data.scalar.style == YAML_SINGLE_QUOTED_SCALAR_STYLE) {
if (len > 9 && isdigit(val[0]) && isdigit(val[1]) && isdigit(val[2]) && isdigit(val[3]) && val[4] == '-'
&& isdigit(val[5]) && isdigit(val[6]) && val[7] == '-'
&& isdigit(val[8]) && isdigit(val[9]))
return parseAbsoluteDate();
// try
AbstractQoreNode* n = try_parse_number(val, len, true);
return n ? n : new QoreStringNode(val, len, QCS_UTF8);
}

// check for boolean values
if (!strcmp(val, "true"))
return &True;
Expand All @@ -329,14 +344,16 @@ AbstractQoreNode* QoreYamlParser::parseScalar(bool favor_string) {

// check for null
if (!strcmp(val, "null") || !strcmp(val, "~") || !len)
return 0;
return nullptr;

// check for sqlnull
if (!strcmp(val, "sqlnull"))
return &Null;

// check for absolute date/time values
if (isdigit(val[0]) && isdigit(val[1]) && isdigit(val[2]) && isdigit(val[3]) && val[4] == '-')
if (len > 9 && isdigit(val[0]) && isdigit(val[1]) && isdigit(val[2]) && isdigit(val[3]) && val[4] == '-'
&& isdigit(val[5]) && isdigit(val[6]) && val[7] == '-'
&& isdigit(val[8]) && isdigit(val[9]))
return parseAbsoluteDate();

// check for relative date/time values (durations)
Expand All @@ -363,14 +380,15 @@ AbstractQoreNode* QoreYamlParser::parseScalar(bool favor_string) {
}

const char* tag = (const char*)event.data.scalar.tag;
// FIXME: use a map here for O(ln(n)) performance
if (!strcmp(tag, YAML_TIMESTAMP_TAG))
return parseAbsoluteDate();
if (!strcmp(tag, YAML_BINARY_TAG))
return parseBase64(val, len, xsink);
if (!strcmp(tag, YAML_STR_TAG))
return new QoreStringNode(val, len, QCS_UTF8);
if (!strcmp(tag, YAML_NULL_TAG))
return 0;
return nullptr;
if (!strcmp(tag, YAML_BOOL_TAG))
return parseBool();
if (!strcmp(tag, YAML_INT_TAG))
Expand All @@ -381,10 +399,12 @@ AbstractQoreNode* QoreYamlParser::parseScalar(bool favor_string) {
return new DateTimeNode(val);
if (!strcmp(tag, QORE_YAML_NUMBER_TAG))
return parseNumber(val, len);
if (!strcmp(tag, QORE_YAML_SQLNULL_TAG))
return &Null;

xsink->raiseException(QY_PARSE_ERR, "don't know how to parse scalar tag '%s'", tag);

return 0;
return nullptr;
}

QoreBoolNode* QoreYamlParser::parseBool() {
Expand Down
11 changes: 6 additions & 5 deletions src/yaml-module.h
View file
Expand Up @@ -211,22 +211,22 @@ class QoreYamlEmitter : public QoreYamlBase {
return emit("map end");
}

DLLLOCAL int emitScalar(const QoreString &value, const char *tag, const char *anchor = 0,
DLLLOCAL int emitScalar(const QoreString &value, const char *tag, const char *anchor = nullptr,
bool plain_implicit = true, bool quoted_implicit = true,
yaml_scalar_style_t style = YAML_ANY_SCALAR_STYLE) {
TempEncodingHelper str(&value, QCS_UTF8, xsink);
if (*xsink)
return -1;

if (!yaml_scalar_event_initialize(&event, (yaml_char_t *)anchor, (yaml_char_t *)tag,
(yaml_char_t *)str->getBuffer(), str->strlen(),
if (!yaml_scalar_event_initialize(&event, (yaml_char_t*)anchor, (yaml_char_t *)tag,
(yaml_char_t*)str->c_str(), str->strlen(),
plain_implicit, quoted_implicit, style))
return err("unknown error initializing yaml scalar output event for yaml type '%s', value '%s'", tag, value.getBuffer());

return emit("scalar", tag);
}

DLLLOCAL int emitScalar(const char *value, const char *tag, const char *anchor = 0,
DLLLOCAL int emitScalar(const char *value, const char *tag, const char *anchor = nullptr,
bool plain_implicit = true, bool quoted_implicit = true,
yaml_scalar_style_t style = YAML_ANY_SCALAR_STYLE) {
if (!yaml_scalar_event_initialize(&event, (yaml_char_t *)anchor, (yaml_char_t *)tag,
Expand Down Expand Up @@ -283,7 +283,8 @@ class QoreYamlEmitter : public QoreYamlBase {
// append precision
tmp.sprintf("{%d}", n.getPrec());
//printd(5, "yaml emit number: %s\n", tmp.getBuffer());
return emitScalar(tmp, QORE_YAML_NUMBER_TAG);
// issue #2343: to avoid ambiguity with single quoted strings, we always use the tag here
return emitScalar(tmp, QORE_YAML_NUMBER_TAG, nullptr, false, false);
}

DLLLOCAL int emitValue(bool b) {
Expand Down
14 changes: 11 additions & 3 deletions test/yaml.qtest
View file
Expand Up @@ -38,6 +38,8 @@ const DATA = (
-@inf@n,
@inf@,
-@inf@,
"'single quoted string'",
"'1234'",
);

const DATA2 = (
Expand All @@ -64,16 +66,15 @@ public class Main inherits QUnit::Test {
addTestCase("Len test", \testLen());
addTestCase("Date/Time test", \testDates());
addTestCase("Structure with NaN test", \testNanStructure());
addTestCase("single quoted strings", \testSingleQuotedStrings());
addTestCase("sql null test", \sqlNull());

# Return for compatibility with test harness that checks return value.
set_return_value(main());
}

testYaml() {
string s = make_yaml(DATA);
any l = parse_yaml(s);
testAssertion("test simple yaml", \equals(), (DATA, l));
assertEq(DATA, parse_yaml(make_yaml(DATA)));
}

testOptions() {
Expand Down Expand Up @@ -122,6 +123,13 @@ public class Main inherits QUnit::Test {
assertEq(DATA2[4], l[4]);
}

testSingleQuotedStrings() {
# issue 2343
assertEq("1234", parse_yaml("'1234'"));
assertEq(500n, parse_yaml("'5e+02n{128}'"));
assertEq(2017-10-26T09:37:36.119613+02:00, parse_yaml("'2017-10-26 09:37:36.119613 +02:00'"));
}

sqlNull() {
string yaml = make_yaml(NULL, EmitSqlNull);
assertRegex("^sqlnull", yaml);
Expand Down

0 comments on commit 61e167b

Please sign in to comment.