Skip to content

/ manticoresearch

Permalink
Browse files

fixed HTTP interface to correctly process query with large size; adde… 

…d status field to bulk reply; added regression to test 351
  • Loading branch information
@tomatolog
tomatolog committed Mar 1, 2018
1 parent 21bfd8f commit caaa3842f4dbd8507fd9e20bf3371a0c8ec0c087
Unified Split
Showing with 110 additions and 2 deletions.
  1. +5 −1 src/searchd.cpp
  2. +1 −1 src/searchdhttp.cpp
  3. +1 −0 src/sphinxjsonquery.cpp
  4. +13 −0 test/test_351/model.bin
  5. +90 −0 test/test_351/test.xml
6 src/searchd.cpp
View file
m_tState->m_iPos += iRes;

// socket would block - going back to polling
if ( iRes==0 )
if ( iRes==0 && m_tState->m_iLeft )
return NE_KEEP;

// keep fetching data till the end of a header
m_tState->m_dBuf.Resize ( iReqSize );
}

// keep reading till end of buffer or data at socket
if ( iRes>0 )
continue;

pLoop->RemoveIterEvent();

sphLogDebugv ( "%p HTTP buf=%d, header=%d, content-len=%d, sock=%d, tick=%u", this, m_tState->m_dBuf.GetLength(), m_tHeadParser.m_iHeaderEnd, m_tHeadParser.m_iFieldContentLenVal, m_iSock, pLoop->m_uTick );
2 src/searchdhttp.cpp
View file
@@ -1137,7 +1137,7 @@ class HttpHandler_JsonBulk_c : public HttpHandler_c, public HttpOptionsTraits_c,
char * p = const_cast<char*>(m_sQuery.cstr());

bool bResult = false;
while ( *p )
while ( p && *p )
{
while ( sphIsSpace(*p) )
p++;
1 src/sphinxjsonquery.cpp
View file
@@ -1970,6 +1970,7 @@ cJSON * sphEncodeInsertResultJson ( const char * szIndex, bool bReplace, SphDocI
cJSON_AddNumberToObject ( pRoot, "_id", tDocId );
cJSON_AddBoolToObject ( pRoot, "created", !bReplace );
cJSON_AddStringToObject ( pRoot, "result", bReplace ? "updated" : "created" );
cJSON_AddNumberToObject ( pRoot, "status", bReplace ? 200 : 201 );

return pRoot;
}
13 test/test_351/model.bin
View file
@@ -0,0 +1,13 @@
a:1:{i:0;a:1:{i:0;a:6:{i:0;s:15:"populate commit";i:1;a:2:{s:4:"rows";s:897:"{"items":[{"create":{"_index":"test_rt","_id":1,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":2,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":3,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":4,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":5,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":6,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":7,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":8,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":9,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":10,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:2;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":11,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":12,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":13,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":14,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":15,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":16,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":17,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":18,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":19,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":20,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:3;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":21,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":22,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":23,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":24,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":25,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":26,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":27,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":28,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":29,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":30,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:4;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":31,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":32,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":33,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":34,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":35,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":36,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":37,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":38,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":39,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":40,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:5;a:5:{s:13:"http_endpoint";s:11:"json/search";s:11:"http_method";s:4:"POST";s:12:"http_request";s:141:"{
"index":"test_rt",
"query":
{
"bool":
{
"must":
[
{ "range": { "id": { "gt": 35 } } }
]
}
}
}";s:4:"rows";s:366:"{"took":0,"timed_out":false,"hits":{"total":5,"hits":[{"_id":"36","_score":1,"_source":{"gid":36,"dummy":"test36"}},{"_id":"37","_score":1,"_source":{"gid":37,"dummy":"test37"}},{"_id":"38","_score":1,"_source":{"gid":38,"dummy":"test38"}},{"_id":"39","_score":1,"_source":{"gid":39,"dummy":"test39"}},{"_id":"40","_score":1,"_source":{"gid":40,"dummy":"test40"}}]}}";s:9:"http_code";i:200;}}}}
90 test/test_351/test.xml
View file
@@ -0,0 +1,90 @@
<?xml version="1.0" encoding="utf-8"?>

<test>
<name>large packet via HTTP</name>

<skip_indexer/>
<requires><http/></requires>

<config>
searchd
{
<searchd_settings/>
binlog_path =
}

index test_rt
{
type = rt
path = <data_path/>/rt
rt_field = title
rt_attr_uint = gid
rt_attr_string = dummy
}
</config>

<custom_test><![CDATA[
global $sd_http_port;
function RandomWordsTest351 ( $n, $seed )
{
srand ( $seed );
$words = [];
for ( $k=0; $k<$n; $k++ )
$words[] = substr ( str_shuffle ( 'abcdefghijklmnopqrstuvwxyz' ), 0, 5 );
return join ( $words, " " );
}
$results = array();
$results[] = 'populate commit';
$con = curl_init();
$id = 1;
for ( $commit=0; $commit<4; $commit++ )
{
$rows = "";
for ( $row=0; $row<10; $row++ )
{
$rows .= json_encode ( array ( "create"=>array("index"=>"test_rt", "id"=>$id, "doc"=>array( "gid"=>$id, "dummy"=>"test".$id, "title"=>RandomWordsTest351 ( 10000, $id ) ))) );
$rows .= "\n";
$id++;
}
$curl_desc = array ( CURLOPT_RETURNTRANSFER => 1, CURLOPT_CONNECTTIMEOUT=>1, CURLOPT_URL => "127.0.0.1:$sd_http_port/json/bulk" );
$curl_desc[CURLOPT_POST] = 1;
$curl_desc[CURLOPT_POSTFIELDS] = $rows;
$curl_desc[CURLOPT_HTTPHEADER] = array("Content-Type: application/x-ndjson", 'Content-Length: ' . strlen($rows));
curl_setopt_array ( $con, $curl_desc );
$res = curl_exec ( $con );
$res = preg_replace('/"time":\d+(\.\d+)*,/', '"time":0.000,', $res);
$http_code = curl_getinfo ( $con, CURLINFO_HTTP_CODE );
$results[] = array ( 'rows'=>$res, 'http_code'=>$http_code );
}
$q = <<<'EOD'
{
"index":"test_rt",
"query":
{
"bool":
{
"must":
[
{ "range": { "id": { "gt": 35 } } }
]
}
}
}
EOD;
$results[] = HttpQueryPost("json/search", NULL, $q );
curl_close($con);
]]></custom_test>

</test>

0 comments on commit caaa384

Please sign in to comment.