fixed HTTP interface to correctly process query with large size; adde…

…d status field to bulk reply; added regression to test 351
manticoresoftware · Mar 1, 2018 · caaa3842f4dbd8507fd9e20bf3371a0c8ec0c087 · caaa384
1 parent 21bfd8f
commit caaa3842f4dbd8507fd9e20bf3371a0c8ec0c087
Unified Split

Showing with 110 additions and 2 deletions.

+5 −1 src/searchd.cpp

+1 −1 src/searchdhttp.cpp

+1 −0 src/sphinxjsonquery.cpp

+13 −0 test/test_351/model.bin

+90 −0 test/test_351/test.xml
diff --git a/src/searchd.cpp b/src/searchd.cpp
@@ -22888,7 +22888,7 @@ NetEvent_e NetReceiveDataHttp_t::Tick ( DWORD uGotEvents, CSphVector<ISphNetActi
 		m_tState->m_iPos += iRes;

 		// socket would block - going back to polling
-		if ( iRes==0 )
+		if ( iRes==0 && m_tState->m_iLeft )
 			return NE_KEEP;

 		// keep fetching data till the end of a header
@@ -22912,6 +22912,10 @@ NetEvent_e NetReceiveDataHttp_t::Tick ( DWORD uGotEvents, CSphVector<ISphNetActi
 			m_tState->m_dBuf.Resize ( iReqSize );
 		}

+		// keep reading till end of buffer or data at socket
+		if ( iRes>0 )
+			continue;
+
 		pLoop->RemoveIterEvent();

 		sphLogDebugv ( "%p HTTP buf=%d, header=%d, content-len=%d, sock=%d, tick=%u", this, m_tState->m_dBuf.GetLength(), m_tHeadParser.m_iHeaderEnd, m_tHeadParser.m_iFieldContentLenVal, m_iSock, pLoop->m_uTick );
diff --git a/src/searchdhttp.cpp b/src/searchdhttp.cpp
@@ -1137,7 +1137,7 @@ class HttpHandler_JsonBulk_c : public HttpHandler_c, public HttpOptionsTraits_c,
 		char * p = const_cast<char*>(m_sQuery.cstr());

 		bool bResult = false;
-		while ( *p )
+		while ( p && *p )
 		{
 			while ( sphIsSpace(*p) )
 				p++;
diff --git a/src/sphinxjsonquery.cpp b/src/sphinxjsonquery.cpp
@@ -1970,6 +1970,7 @@ cJSON * sphEncodeInsertResultJson ( const char * szIndex, bool bReplace, SphDocI
 	cJSON_AddNumberToObject ( pRoot, "_id", tDocId );
 	cJSON_AddBoolToObject ( pRoot, "created", !bReplace );
 	cJSON_AddStringToObject ( pRoot, "result", bReplace ? "updated" : "created" );
+	cJSON_AddNumberToObject ( pRoot, "status", bReplace ? 200 : 201 );

 	return pRoot;
 }
diff --git a/test/test_351/model.bin b/test/test_351/model.bin
@@ -0,0 +1,13 @@
+a:1:{i:0;a:1:{i:0;a:6:{i:0;s:15:"populate commit";i:1;a:2:{s:4:"rows";s:897:"{"items":[{"create":{"_index":"test_rt","_id":1,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":2,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":3,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":4,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":5,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":6,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":7,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":8,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":9,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":10,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:2;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":11,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":12,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":13,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":14,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":15,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":16,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":17,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":18,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":19,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":20,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:3;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":21,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":22,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":23,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":24,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":25,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":26,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":27,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":28,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":29,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":30,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:4;a:2:{s:4:"rows";s:906:"{"items":[{"create":{"_index":"test_rt","_id":31,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":32,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":33,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":34,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":35,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":36,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":37,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":38,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":39,"created":true,"result":"created","status":201}},{"create":{"_index":"test_rt","_id":40,"created":true,"result":"created","status":201}}],"errors":false}";s:9:"http_code";i:200;}i:5;a:5:{s:13:"http_endpoint";s:11:"json/search";s:11:"http_method";s:4:"POST";s:12:"http_request";s:141:"{
+  "index":"test_rt",
+  "query":
+  {
+    "bool":
+    {
+      "must":
+      [
+        { "range": { "id": { "gt": 35 } } }
+      ]
+    }
+  }
+}";s:4:"rows";s:366:"{"took":0,"timed_out":false,"hits":{"total":5,"hits":[{"_id":"36","_score":1,"_source":{"gid":36,"dummy":"test36"}},{"_id":"37","_score":1,"_source":{"gid":37,"dummy":"test37"}},{"_id":"38","_score":1,"_source":{"gid":38,"dummy":"test38"}},{"_id":"39","_score":1,"_source":{"gid":39,"dummy":"test39"}},{"_id":"40","_score":1,"_source":{"gid":40,"dummy":"test40"}}]}}";s:9:"http_code";i:200;}}}}
diff --git a/test/test_351/test.xml b/test/test_351/test.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<test>
+<name>large packet via HTTP</name>
+
+<skip_indexer/>
+<requires><http/></requires>
+
+<config>
+searchd
+{
+	<searchd_settings/>
+	binlog_path =
+}
+
+index test_rt
+{
+    type = rt
+    path = <data_path/>/rt
+    rt_field = title
+    rt_attr_uint = gid
+	rt_attr_string = dummy
+}
+</config>
+
+<custom_test><![CDATA[
+
+global $sd_http_port;
+
+function RandomWordsTest351 ( $n, $seed )
+{
+	srand ( $seed );
+	$words = [];
+	for ( $k=0; $k<$n; $k++ )
+		$words[] = substr ( str_shuffle ( 'abcdefghijklmnopqrstuvwxyz' ), 0, 5 );
+	return join ( $words, " " );
+}
+
+$results = array();
+$results[] = 'populate commit';
+
+$con = curl_init();
+	
+$id = 1;
+for ( $commit=0; $commit<4; $commit++ )
+{
+	$rows = "";
+	for ( $row=0; $row<10; $row++ )
+	{
+		$rows .= json_encode ( array ( "create"=>array("index"=>"test_rt", "id"=>$id, "doc"=>array( "gid"=>$id, "dummy"=>"test".$id, "title"=>RandomWordsTest351 ( 10000, $id ) ))) );
+		$rows .= "\n";
+		$id++;
+	}
+	
+	$curl_desc = array ( CURLOPT_RETURNTRANSFER => 1, CURLOPT_CONNECTTIMEOUT=>1, CURLOPT_URL => "127.0.0.1:$sd_http_port/json/bulk" );
+	$curl_desc[CURLOPT_POST] = 1;
+	$curl_desc[CURLOPT_POSTFIELDS] = $rows;
+	$curl_desc[CURLOPT_HTTPHEADER] = array("Content-Type: application/x-ndjson", 'Content-Length: ' . strlen($rows));
+	
+	curl_setopt_array ( $con, $curl_desc );
+	$res = curl_exec ( $con );
+	$res = preg_replace('/"time":\d+(\.\d+)*,/', '"time":0.000,', $res);
+	$http_code = curl_getinfo ( $con, CURLINFO_HTTP_CODE );
+	
+	$results[] = array ( 'rows'=>$res, 'http_code'=>$http_code );
+}
+
+$q = <<<'EOD'
+{
+  "index":"test_rt",
+  "query":
+  {
+    "bool":
+    {
+      "must":
+      [
+        { "range": { "id": { "gt": 35 } } }
+      ]
+    }
+  }
+}
+EOD;
+
+$results[] = HttpQueryPost("json/search", NULL, $q );
+
+curl_close($con);
+
+]]></custom_test>
+
+</test>