Skip to content
Permalink
Browse files

fixed #1080 JSON converts only numeric to string; fixed JSON string t…

…o numeric conversion at expressions; added case as test 396; fixed model at test 253
  • Loading branch information
tomatolog committed Nov 18, 2019
1 parent 6e3fc9e commit faed3220b423a96401982bf47117edf1f62c584e
Showing with 106 additions and 42 deletions.
  1. +39 −41 src/sphinxexpr.cpp
  2. +7 −0 src/sphinxjson.cpp
  3. +1 −1 test/test_253/model.bin
  4. +1 −0 test/test_396/model.bin
  5. +58 −0 test/test_396/test.xml
@@ -1537,8 +1537,6 @@ class Expr_JsonFieldConv_c : public ISphExpr
case JSON_TRUE: return 1;
case JSON_STRING:
{
if ( !g_bJsonAutoconvNumbers )
return 0;
int iLen = sphJsonUnpackInt ( &pVal );
int64_t iVal;
double fVal;
@@ -3109,32 +3107,32 @@ static int FuncHashLookup ( const char * sKey )

static BYTE dAsso[] =
{
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
40, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 5, 124, 15, 5, 5,
65, 0, 85, 20, 50, 0, 124, 124, 20, 0,
10, 0, 40, 25, 5, 40, 0, 25, 124, 70,
60, 40, 0, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
40, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 5, 124, 15, 5, 5,
65, 0, 85, 20, 50, 0, 124, 124, 20, 0,
10, 0, 40, 25, 5, 40, 0, 25, 124, 70,
60, 40, 0, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124, 124, 124, 124, 124,
124, 124, 124, 124, 124, 124
};

auto * s = (const BYTE*) sKey;
@@ -3148,19 +3146,19 @@ static int FuncHashLookup ( const char * sKey )

static int dIndexes[] =
{
-1, -1, -1, -1, -1, -1, -1, -1, 56, 2,
33, 32, 31, 23, 41, 16, 21, 46, 30, -1,
13, 40, 39, 38, 62, 63, -1, 34, 58, 37,
66, 11, 6, 50, 65, 64, 48, -1, 54, 51,
49, 42, 53, 55, 7, 8, -1, 57, 5, 29,
45, 22, -1, 4, 12, 52, -1, -1, 59, 17,
-1, -1, -1, 1, 18, 35, 36, 19, 60, 26,
-1, -1, -1, 43, 10, -1, -1, -1, 24, 20,
-1, -1, 61, 0, 28, 67, -1, 27, -1, 68,
-1, -1, -1, -1, -1, -1, 47, -1, -1, 14,
-1, -1, -1, 9, -1, -1, -1, -1, -1, -1,
3, -1, 44, 25, -1, -1, -1, -1, -1, -1,
-1, -1, -1, 15,
-1, -1, -1, -1, -1, -1, -1, -1, 56, 2,
33, 32, 31, 23, 41, 16, 21, 46, 30, -1,
13, 40, 39, 38, 62, 63, -1, 34, 58, 37,
66, 11, 6, 50, 65, 64, 48, -1, 54, 51,
49, 42, 53, 55, 7, 8, -1, 57, 5, 29,
45, 22, -1, 4, 12, 52, -1, -1, 59, 17,
-1, -1, -1, 1, 18, 35, 36, 19, 60, 26,
-1, -1, -1, 43, 10, -1, -1, -1, 24, 20,
-1, -1, 61, 0, 28, 67, -1, 27, -1, 68,
-1, -1, -1, -1, -1, -1, 47, -1, -1, 14,
-1, -1, -1, 9, -1, -1, -1, -1, -1, -1,
3, -1, 44, 25, -1, -1, -1, -1, -1, -1,
-1, -1, -1, 15,
};

if ( iHash>=(int)(sizeof(dIndexes)/sizeof(dIndexes[0])) )
@@ -1274,6 +1274,13 @@ bool sphJsonStringToNumber ( const char * s, int iLen, ESphJsonType &eType, int6
if ( !iLen || iLen==64 )
return false;

const char * pCheck = s;
// check string conversion is valid if only whitespaces left
while ( pCheck<sEnd && is_json_whitespace(*pCheck) )
++pCheck;
if ( pCheck!=sEnd )
return false;

char * pTail;
errno = 0;
if ( !bIsFloat )

Large diffs are not rendered by default.

@@ -0,0 +1 @@
a:1:{i:0;a:3:{i:0;a:3:{s:8:"sphinxql";s:101:"select id, json_col j, all(x>1 and x<7 for x in j.a) a from test where match('title') order by id asc";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:3:{s:2:"id";s:1:"1";s:1:"j";s:36:"{"a":[1,2,3,4],"t":["t1","t2","t3"]}";s:1:"a";s:1:"0";}i:1;a:3:{s:2:"id";s:1:"2";s:1:"j";s:36:"{"a":[2,3,4,5],"t":["t2","t3","t4"]}";s:1:"a";s:1:"1";}i:2;a:3:{s:2:"id";s:1:"3";s:1:"j";s:36:"{"a":[3,4,5,1],"t":["t3","t4","t5"]}";s:1:"a";s:1:"0";}i:3;a:3:{s:2:"id";s:1:"4";s:1:"j";s:44:"{"a":["4","5","6","2"],"t":["t4","t5","t6"]}";s:1:"a";s:1:"1";}i:4;a:3:{s:2:"id";s:1:"5";s:1:"j";s:38:"{"a":["4","5all","test 6","2"],"b":[]}";s:1:"a";s:1:"0";}i:5;a:3:{s:2:"id";s:1:"6";s:1:"j";s:15:"{"a":[],"b":[]}";s:1:"a";s:1:"0";}}}i:1;a:3:{s:8:"sphinxql";s:101:"select id, json_col j, json_col.a jc, in(j.a,5,6) as a from test where match('title') order by id asc";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:1:"j";s:36:"{"a":[1,2,3,4],"t":["t1","t2","t3"]}";s:2:"jc";s:9:"[1,2,3,4]";s:1:"a";s:1:"0";}i:1;a:4:{s:2:"id";s:1:"2";s:1:"j";s:36:"{"a":[2,3,4,5],"t":["t2","t3","t4"]}";s:2:"jc";s:9:"[2,3,4,5]";s:1:"a";s:1:"1";}i:2;a:4:{s:2:"id";s:1:"3";s:1:"j";s:36:"{"a":[3,4,5,1],"t":["t3","t4","t5"]}";s:2:"jc";s:9:"[3,4,5,1]";s:1:"a";s:1:"1";}i:3;a:4:{s:2:"id";s:1:"4";s:1:"j";s:44:"{"a":["4","5","6","2"],"t":["t4","t5","t6"]}";s:2:"jc";s:17:"["4","5","6","2"]";s:1:"a";s:1:"0";}i:4;a:4:{s:2:"id";s:1:"5";s:1:"j";s:38:"{"a":["4","5all","test 6","2"],"b":[]}";s:2:"jc";s:25:"["4","5all","test 6","2"]";s:1:"a";s:1:"0";}i:5;a:4:{s:2:"id";s:1:"6";s:1:"j";s:15:"{"a":[],"b":[]}";s:2:"jc";s:2:"[]";s:1:"a";s:1:"0";}}}i:2;a:3:{s:8:"sphinxql";s:115:"select id, json_col j, json_col.a jc, any(x=5 or x=6 for x in j.a) a from test where match('title') order by id asc";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:1:"j";s:36:"{"a":[1,2,3,4],"t":["t1","t2","t3"]}";s:2:"jc";s:9:"[1,2,3,4]";s:1:"a";s:1:"0";}i:1;a:4:{s:2:"id";s:1:"2";s:1:"j";s:36:"{"a":[2,3,4,5],"t":["t2","t3","t4"]}";s:2:"jc";s:9:"[2,3,4,5]";s:1:"a";s:1:"1";}i:2;a:4:{s:2:"id";s:1:"3";s:1:"j";s:36:"{"a":[3,4,5,1],"t":["t3","t4","t5"]}";s:2:"jc";s:9:"[3,4,5,1]";s:1:"a";s:1:"1";}i:3;a:4:{s:2:"id";s:1:"4";s:1:"j";s:44:"{"a":["4","5","6","2"],"t":["t4","t5","t6"]}";s:2:"jc";s:17:"["4","5","6","2"]";s:1:"a";s:1:"1";}i:4;a:4:{s:2:"id";s:1:"5";s:1:"j";s:38:"{"a":["4","5all","test 6","2"],"b":[]}";s:2:"jc";s:25:"["4","5all","test 6","2"]";s:1:"a";s:1:"0";}i:5;a:4:{s:2:"id";s:1:"6";s:1:"j";s:15:"{"a":[],"b":[]}";s:2:"jc";s:2:"[]";s:1:"a";s:1:"0";}}}}}
@@ -0,0 +1,58 @@
<?xml version="1.0" encoding="utf-8"?>
<test>

<name>json string to number conversion</name>

<config>
indexer
{
mem_limit = 16M
}

searchd
{
<searchd_settings/>
}

source test
{
type = mysql
<sql_settings/>
sql_query = select * from test_table
sql_attr_uint = gid
sql_attr_json = json_col
}

index test
{
source = test
path = <data_path/>/test
}
</config>

<db_create>
CREATE TABLE test_table
(
id INTEGER PRIMARY KEY NOT NULL,
gid INTEGER NOT NULL,
json_col VARCHAR(255) NOT NULL,
title VARCHAR(255) NOT NULL
);
</db_create>
<db_drop>DROP TABLE IF EXISTS test_table;</db_drop>
<db_insert>INSERT INTO test_table VALUES
( 1, 123, '{"a":[1,2,3,4], "t":["t1", "t2", "t3"]}', 'title' ),
( 2, 234, '{"a":[2,3,4,5], "t":["t2", "t3", "t4"]}', 'title' ),
( 3, 345, '{"a":[3,4,5,1], "t":["t3", "t4", "t5"]}', 'title' ),
( 4, 456, '{"a":["4","5","6","2"], "t":["t4", "t5", "t6"]}','title' ),
( 5, 567, '{"a":["4","5all","test 6","2"], "b":[]}', 'title' ),
( 6, 678, '{"a":[],"b":[]}', 'title' )
</db_insert>

<sphqueries>
<sphinxql>select id, json_col j, all(x&gt;1 and x&lt;7 for x in j.a) a from test where match('title') order by id asc</sphinxql>
<sphinxql>select id, json_col j, json_col.a jc, in(j.a,5,6) as a from test where match('title') order by id asc</sphinxql>
<sphinxql>select id, json_col j, json_col.a jc, any(x=5 or x=6 for x in j.a) a from test where match('title') order by id asc</sphinxql>
</sphqueries>

</test>

0 comments on commit faed322

Please sign in to comment.
You can’t perform that action at this time.