Skip to content
Permalink
Browse files

fixed #735 crash of daemon due to string at sorter from RT index with…

… disk chunks
  • Loading branch information...
tomatolog committed Mar 6, 2019
1 parent c2f4af1 commit 4aa6c69ad3a181cd7c5dafb990fb528121f68e59
Showing with 39 additions and 21 deletions.
  1. +20 −2 src/sphinx.cpp
  2. +2 −2 src/sphinx.h
  3. +1 −0 src/sphinxint.h
  4. +1 −1 src/sphinxrt.cpp
  5. +15 −16 src/sphinxsort.cpp
@@ -8573,6 +8573,11 @@ MatchesToNewSchema_c::MatchesToNewSchema_c ( const ISphSchema * pOldSchema, cons
m_dOld2New.Resize ( m_pOldSchema->GetAttrsCount() );
for ( int i=0; i < m_pOldSchema->GetAttrsCount(); i++ )
m_dOld2New[i] = m_pNewSchema->GetAttrIndex ( m_pOldSchema->GetAttr(i).m_sName.cstr() );

// need to update @int_str2ptr_ locator to use new schema
// no need to pass pOldSchema as we remapping only new schema pointers
// also need to update group sorter keypart to be str_ptr in caller code SetSchema
sphSortGetStringRemap ( *pNewSchema, *pNewSchema, m_dRemapStringCmp );
}


@@ -8659,6 +8664,9 @@ void MatchesToNewSchema_c::Process ( CSphMatch * pMatch )
for ( const auto & i : m_dNewAttrs )
tResult.SetAttr ( i, 0 );

for ( const SphStringSorterRemap_t & tRemap : m_dRemapStringCmp )
tResult.SetAttr ( tRemap.m_tDst, tResult.GetAttr ( tRemap.m_tSrc ) );

Swap ( pMatch->m_pDynamic, tResult.m_pDynamic );
pMatch->m_pStatic = nullptr;
}
@@ -8742,7 +8750,7 @@ static void PooledAttrsToPtrAttrs ( ISphMatchSorter ** ppSorters, int nSorters,
DiskMatchesToNewSchema_c fnFinal ( pOldSchema, pNewSchema, pMVA, pStrings, bArenaProhibit );
pSorter->Finalize ( fnFinal, false );

pSorter->SetSchema ( pNewSchema );
pSorter->SetSchema ( pNewSchema, true );
SafeDelete ( pOldSchema );
}
}
@@ -14284,10 +14292,20 @@ SphWordID_t CSphDictExact::GetWordID ( BYTE * pWord )

/////////////////////////////////////////////////////////////////////////////

void CSphMatchComparatorState::FixupLocators ( const ISphSchema * pOldSchema, const ISphSchema * pNewSchema )
void CSphMatchComparatorState::FixupLocators ( const ISphSchema * pOldSchema, const ISphSchema * pNewSchema, bool bRemapKeyparts )
{
for ( auto & i : m_tLocator )
sphFixupLocator ( i, pOldSchema, pNewSchema );

// update string keypart into str_ptr
if ( bRemapKeyparts )
{
for ( int iAttr=0; iAttr<MAX_ATTRS; iAttr++ )
{
if ( m_eKeypart[iAttr]==SPH_KEYPART_STRING )
m_eKeypart[iAttr] = SPH_KEYPART_STRINGPTR;
}
}
}


@@ -3031,7 +3031,7 @@ struct CSphMatchComparatorState
return m_fnStrCmp ( aa, bb, eStrSource, 0, 0 );
}

void FixupLocators ( const ISphSchema * pOldSchema, const ISphSchema * pNewSchema );
void FixupLocators ( const ISphSchema * pOldSchema, const ISphSchema * pNewSchema, bool bRemapKeyparts );
};


@@ -3090,7 +3090,7 @@ class ISphMatchSorter
virtual void SetStringPool ( const BYTE * ) {}

/// set sorter schema
virtual void SetSchema ( ISphSchema * pSchema );
virtual void SetSchema ( ISphSchema * pSchema, bool bRemapCmp );

/// get incoming schema
virtual const ISphSchema * GetSchema () const { return m_pSchema; }
@@ -2472,6 +2472,7 @@ class MatchesToNewSchema_c : public ISphMatchProcessor
const ISphSchema * m_pNewSchema;
CSphVector<CSphAttrLocator> m_dNewAttrs;
CSphVector<int> m_dOld2New;
CSphVector<SphStringSorterRemap_t> m_dRemapStringCmp;

virtual const DWORD * GetMVAPool ( const CSphMatch * pMatch ) = 0;
virtual const BYTE * GetStringPool ( const CSphMatch * pMatch ) = 0;
@@ -6682,7 +6682,7 @@ static void TransformSorterSchema ( ISphMatchSorter * pSorter, const SphChunkGua
RTMatchesToNewSchema_c fnFinal ( pOldSchema, pNewSchema, tGuard, dMVA, dStrings, tMvaArenaFlag );
pSorter->Finalize ( fnFinal, false );

pSorter->SetSchema ( pNewSchema );
pSorter->SetSchema ( pNewSchema, true );
SafeDelete ( pOldSchema );
}

@@ -57,12 +57,10 @@ ISphMatchSorter::~ISphMatchSorter()
}


void ISphMatchSorter::SetSchema ( ISphSchema * pSchema )
void ISphMatchSorter::SetSchema ( ISphSchema * pSchema, bool bRemapCmp )
{
assert ( pSchema );

for ( auto & i : m_tState.m_tLocator )
sphFixupLocator ( i, m_pSchema, pSchema );
m_tState.FixupLocators ( m_pSchema, pSchema, bRemapCmp );

m_pSchema = pSchema;
}
@@ -1691,15 +1689,15 @@ static SphAttr_t GetDistinctKey ( const CSphMatch & tEntry, const CSphAttrLocato
template <typename COMPGROUP>
static void FixupSorterLocators ( CSphGroupSorterSettings & tSettings, ISphSchema * pOldSchema,
const ISphSchema * pNewSchema, GroupSorter_fn<COMPGROUP> * pGroupSorter,
CSphVector<IAggrFunc *> & dAggregates, MatchCloner_t & tPregroup )
CSphVector<IAggrFunc *> & dAggregates, MatchCloner_t & tPregroup, bool bRemapCmp )
{
// this is the case when we are moving to a new schema with non-pooled attrs
if ( !pOldSchema )
return;

tSettings.FixupLocators ( pOldSchema, pNewSchema );
if ( pGroupSorter )
pGroupSorter->FixupLocators ( pOldSchema, pNewSchema );
pGroupSorter->FixupLocators ( pOldSchema, pNewSchema, bRemapCmp );

for ( auto i : dAggregates )
SafeDelete(i);
@@ -1870,10 +1868,10 @@ class CSphKBufferGroupSorter : public CSphMatchQueueTraits, protected BaseGroupS
}

/// schema setup
void SetSchema ( ISphSchema * pSchema ) override
void SetSchema ( ISphSchema * pSchema, bool bRemapCmp ) override
{
FixupSorterLocators ( *this, m_pSchema, pSchema, &m_tGroupSorter, m_dAggregates, m_tPregroup );
ISphMatchSorter::SetSchema ( pSchema );
FixupSorterLocators ( *this, m_pSchema, pSchema, &m_tGroupSorter, m_dAggregates, m_tPregroup, bRemapCmp );
ISphMatchSorter::SetSchema ( pSchema, bRemapCmp );
m_dAvgs.Resize ( 0 );
SetupBaseGrouper<DISTINCT> ( pSchema, m_tGroupSorter.m_eKeypart, m_tGroupSorter.m_tLocator, &m_dAvgs );
}
@@ -2317,10 +2315,10 @@ class CSphKBufferNGroupSorter : public CSphMatchQueueTraits, protected BaseGroup
}

/// schema setup
void SetSchema ( ISphSchema * pSchema ) override
void SetSchema ( ISphSchema * pSchema, bool bRemapCmp ) override
{
FixupSorterLocators ( *this, m_pSchema, pSchema, &m_tGroupSorter, m_dAggregates, m_tPregroup );
ISphMatchSorter::SetSchema ( pSchema );
FixupSorterLocators ( *this, m_pSchema, pSchema, &m_tGroupSorter, m_dAggregates, m_tPregroup, bRemapCmp );
ISphMatchSorter::SetSchema ( pSchema, bRemapCmp );
m_dAvgs.Resize ( 0 );
SetupBaseGrouper<DISTINCT> ( pSchema, m_tGroupSorter.m_eKeypart, m_tGroupSorter.m_tLocator, &m_dAvgs );
}
@@ -3235,10 +3233,10 @@ class CSphImplicitGroupSorter : public ISphMatchSorter, ISphNoncopyable, protect
}

/// schema setup
void SetSchema ( ISphSchema * pSchema ) override
void SetSchema ( ISphSchema * pSchema, bool bRemapCmp ) override
{
FixupSorterLocators<void> ( *this, m_pSchema, pSchema, nullptr, m_dAggregates, m_tPregroup );
ISphMatchSorter::SetSchema ( pSchema );
FixupSorterLocators<void> ( *this, m_pSchema, pSchema, nullptr, m_dAggregates, m_tPregroup, bRemapCmp );
ISphMatchSorter::SetSchema ( pSchema, bRemapCmp );
SetupBaseGrouper<DISTINCT> ( pSchema );
}

@@ -4649,6 +4647,7 @@ static void SetupSortRemap ( CSphRsetSchema & tSorterSchema, CSphMatchComparator
tSorterSchema.AddAttr ( tRemapCol, true );
}
tState.m_tLocator[i] = tSorterSchema.GetAttr ( iRemap ).m_tLocator;
tState.m_dAttrs[i] = iRemap;
}
}

@@ -5781,7 +5780,7 @@ ISphMatchSorter * sphCreateQueue ( SphQueueSettings_t & tQueue )
assert ( pTop );
pTop->SetState ( tStateMatch );
pTop->SetGroupState ( tStateGroup );
pTop->SetSchema ( pSorterSchema.LeakPtr() );
pTop->SetSchema ( pSorterSchema.LeakPtr(), false );
pTop->m_bRandomize = bRandomize;

if ( bRandomize )

0 comments on commit 4aa6c69

Please sign in to comment.
You can’t perform that action at this time.