Skip to content
Permalink
Browse files

fixed effyis#14 some hits got lost at batch insert with document dupl…

…icates; added regressions to test 356
  • Loading branch information
tomatolog committed Apr 27, 2018
1 parent 2705de5 commit 705d8c5f407726ed1c6f9745ecffa375682c8969
Showing with 53 additions and 0 deletions.
  1. +3 −0 src/sphinxrt.cpp
  2. +1 −0 test/test_356/model.bin
  3. +49 −0 test/test_356/test.xml
@@ -1927,6 +1927,7 @@ void RtAccum_t::AddDocument ( ISphHits * pHits, const CSphMatch & tDoc, bool bRe

iHits = pHits->Length();
m_dAccum.Reserve ( m_dAccum.GetLength()+iHits );
iHits = 0;
for ( const CSphWordHit * pHit = pHits->First(); pHit<=pHits->Last(); pHit++ )
{
// ignore duplicate hits
@@ -1940,10 +1941,12 @@ void RtAccum_t::AddDocument ( ISphHits * pHits, const CSphMatch & tDoc, bool bRe
// accumulate
m_dAccum.Add ( *pHit );
tLastHit = *pHit;
iHits++;
}
if ( pFieldLens )
pFieldLens [ HITMAN::GetField ( tLastHit.m_uWordPos ) ] = HITMAN::GetPos ( tLastHit.m_uWordPos );
}
// make sure to get real count without duplicated hits
m_dPerDocHitsCount.Add ( iHits );

m_iAccumDocs++;
@@ -0,0 +1 @@
a:1:{i:0;a:5:{i:0;a:2:{s:8:"sphinxql";s:908:"replace into rt (id, gid, body1, body2, body3) values (1, 10, 'the test', 'and the best', 'some share'), (2, 10, 'which he leaned', 'against the tree', 'and share'), (3, 10, 'appeared to think', ' deeply for a moment', 'but share'), (4, 10, 'to put the oil-can', 'in the basket', 'share'), (5, 10, 'various ancient traditions refer', 'to a lost antediluvian world', 'share'), (61, 10, 'many means of transportation all', 'over India and Nepal', 'share'), (61, 10, 'many means of transportation all', 'but India and Nepal', 'share'), (61, 10, 'many means of transportation all', 'either India or Nepal', 'share'), (61, 10, 'many means of transportation all', 'throu India and Nepal', 'share'), (71, 10, 'which can be called', 'due to', 'share'), (72, 10, 'was bothered by my hard', 'look and feel', 'share'), (73, 10, 'wife and man', 'and people', 'share'), (74, 10, 'child and woman', 'with family', 'share')";s:14:"total_affected";i:13;}i:1;a:3:{s:8:"sphinxql";s:48:"select * from rt where match ('share') and id=71";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:2:"71";s:3:"gid";s:2:"10";}}}i:2;a:3:{s:8:"sphinxql";s:48:"select * from rt where match ('share') and id=72";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:2:"72";s:3:"gid";s:2:"10";}}}i:3;a:3:{s:8:"sphinxql";s:48:"select * from rt where match ('share') and id=73";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:2:"73";s:3:"gid";s:2:"10";}}}i:4;a:3:{s:8:"sphinxql";s:48:"select * from rt where match ('share') and id=74";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:2:"id";s:2:"74";s:3:"gid";s:2:"10";}}}}}
@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="utf-8"?>
<test>

<name>duplicate supress hits at another documents</name>

<skip_indexer/>
<requires>
<lemmatizer_base/>
</requires>

<config>
common
{
lemmatizer_base = <local>lemmatizer_base</local>
}

searchd
{
<searchd_settings/>
workers = threads
}

index rt
{
type = rt
docinfo = extern
path = <data_path/>/rt
dict = keywords
morphology = lemmatize_ru_all, lemmatize_en_all
charset_table = english, _, 0..9, russian

rt_attr_uint = gid
rt_field = body1
rt_field = body2
rt_field = body3

rt_mem_limit = 8M
}
</config>

<sphqueries>
<sphinxql>replace into rt (id, gid, body1, body2, body3) values (1, 10, 'the test', 'and the best', 'some share'), (2, 10, 'which he leaned', 'against the tree', 'and share'), (3, 10, 'appeared to think', ' deeply for a moment', 'but share'), (4, 10, 'to put the oil-can', 'in the basket', 'share'), (5, 10, 'various ancient traditions refer', 'to a lost antediluvian world', 'share'), (61, 10, 'many means of transportation all', 'over India and Nepal', 'share'), (61, 10, 'many means of transportation all', 'but India and Nepal', 'share'), (61, 10, 'many means of transportation all', 'either India or Nepal', 'share'), (61, 10, 'many means of transportation all', 'throu India and Nepal', 'share'), (71, 10, 'which can be called', 'due to', 'share'), (72, 10, 'was bothered by my hard', 'look and feel', 'share'), (73, 10, 'wife and man', 'and people', 'share'), (74, 10, 'child and woman', 'with family', 'share')</sphinxql>
<sphinxql>select * from rt where match ('share') and id=71</sphinxql>
<sphinxql>select * from rt where match ('share') and id=72</sphinxql>
<sphinxql>select * from rt where match ('share') and id=73</sphinxql>
<sphinxql>select * from rt where match ('share') and id=74</sphinxql>
</sphqueries>

</test>

0 comments on commit 705d8c5

Please sign in to comment.
You can’t perform that action at this time.