Skip to content
Permalink
Browse files
Special free/copy for dynamic ptrs in matches
This is part of the work on memory leak issue in grouper
  • Loading branch information
klirichek committed Sep 6, 2018
1 parent 31db660 commit 56fdbc9ab38973a9a0ba8ccee45f71cf33c16423
Showing with 195 additions and 191 deletions.
  1. +6 −3 src/searchd.cpp
  2. +49 −48 src/sphinx.cpp
  3. +22 −15 src/sphinx.h
  4. +118 −125 src/sphinxsort.cpp
@@ -5231,25 +5231,28 @@ void RemapResult ( const ISphSchema * pTarget, AggrResult_t * pRes )
{
int iCur = 0;
CSphVector<int> dMapFrom ( pTarget->GetAttrsCount() );
CSphVector<int> dRowItems ( pTarget->GetAttrsCount () );
static const int SIZE_OF_ROW = 8 * sizeof ( CSphRowitem );

ARRAY_FOREACH ( iSchema, pRes->m_dSchemas )
{
dMapFrom.Resize ( 0 );
dRowItems.Resize ( 0 );
CSphSchema & dSchema = pRes->m_dSchemas[iSchema];
for ( int i=0; i<pTarget->GetAttrsCount(); i++ )
{
auto iSrcCol = dSchema.GetAttrIndex ( pTarget->GetAttr ( i ).m_sName.cstr () );
const CSphColumnInfo &tSrcCol = dSchema.GetAttr ( iSrcCol );
dMapFrom.Add ( iSrcCol );
dRowItems.Add ( tSrcCol.m_tLocator.m_iBitOffset / SIZE_OF_ROW );
assert ( dMapFrom[i]>=0
|| pTarget->GetAttr(i).m_tLocator.IsID()
|| sphIsSortStringInternal ( pTarget->GetAttr(i).m_sName.cstr() )
|| pTarget->GetAttr(i).m_sName=="@groupbystr"
);
int iOffset = tSrcCol.m_tLocator.m_iBitOffset / ( 8 * sizeof ( CSphRowitem ) );
dSchema.DiscardPtr ( iOffset );
}
int iLimit = Min ( iCur + pRes->m_dMatchCounts[iSchema], pRes->m_dMatches.GetLength() );
dSchema.SubsetPtrs ( dRowItems );
for ( int i=iCur; i<iLimit; i++ )
{
CSphMatch & tMatch = pRes->m_dMatches[i];
@@ -5285,7 +5288,7 @@ void RemapResult ( const ISphSchema * pTarget, AggrResult_t * pRes )
}
// swap out old (most likely wrong sized) match
Swap ( tMatch, tRow );
dSchema.FreeDataPtrs ( &tRow );
dSchema.FreeDataSpecial ( &tRow, dRowItems );
}

iCur = iLimit;
@@ -5989,67 +5989,85 @@ void CSphSchemaHelper::InsertAttr ( CSphVector<CSphColumnInfo> & dAttrs, CSphVec
void CSphSchemaHelper::Reset()
{
m_dDataPtrAttrs.Reset();
m_dDynamicUsed.Reset ();
}

void CSphSchemaHelper::CloneMatch ( CSphMatch * pDst, const CSphMatch & rhs ) const
{
CloneMatchSpecial ( pDst, rhs, m_dDataPtrAttrs );
}

void CSphSchemaHelper::FreeDataPtrs ( CSphMatch * pMatch ) const
{
assert ( pMatch );
if ( !pMatch->m_pDynamic )
return;

for ( auto iOffset : m_dDataPtrAttrs )
{
BYTE * pData = *(BYTE**)(pMatch->m_pDynamic+iOffset);
if ( pData )
{
delete [] pData;
*(BYTE**)(pMatch->m_pDynamic+iOffset) = NULL;
}
}
FreeDataSpecial ( pMatch, m_dDataPtrAttrs );
}

void CSphSchemaHelper::DiscardPtr ( int iAttr )
void CSphSchemaHelper::CopyPtrs ( CSphMatch * pDst, const CSphMatch &rhs ) const
{
m_dDataPtrAttrs.RemoveValue ( iAttr );
CopyDataSpecial ( pDst, rhs, m_dDataPtrAttrs );
}

void CSphSchemaHelper::SubsetPtrs ( CSphVector<int> &dDiscarded )
{
CSphVector<int> dFiltered;
dDiscarded.Uniq ();
for ( int iPtr : m_dDataPtrAttrs )
if ( !dDiscarded.BinarySearch ( iPtr ) )
dFiltered.Add ( iPtr );
dFiltered.Uniq ();
dDiscarded.SwapData ( dFiltered );
}

void CSphSchemaHelper::CloneMatch ( CSphMatch * pDst, const CSphMatch & rhs ) const
void CSphSchemaHelper::CloneMatchSpecial ( CSphMatch * pDst, const CSphMatch &rhs, const CSphVector<int> &dSpecials ) const
{
assert ( pDst );
FreeDataPtrs ( pDst );
pDst->Combine ( rhs, GetDynamicSize() );
CopyPtrs ( pDst, rhs );
FreeDataSpecial ( pDst, dSpecials );
pDst->Combine ( rhs, GetDynamicSize () );
CopyDataSpecial ( pDst, rhs, dSpecials );
}

void CSphSchemaHelper::FreeDataSpecial ( CSphMatch * pMatch, const CSphVector<int> &dSpecials )
{
assert ( pMatch );
if ( !pMatch->m_pDynamic )
return;

for ( auto iOffset : dSpecials )
{
BYTE * pData = *( BYTE ** ) ( pMatch->m_pDynamic + iOffset );
if ( pData )
{
delete[] pData;
*( BYTE ** ) ( pMatch->m_pDynamic + iOffset ) = NULL;
}
}
}

void CSphSchemaHelper::CopyPtrs ( CSphMatch * pDst, const CSphMatch & rhs ) const
void CSphSchemaHelper::CopyDataSpecial ( CSphMatch * pDst, const CSphMatch &rhs, const CSphVector<int> &dSpecials )
{
// notes on PACKEDFACTORS
// not immediately obvious: this is not needed while pushing matches to sorters; factors are held in an outer hash table
// but it is necessary to copy factors when combining results from several indexes via a sorter because at this moment matches are the owners of factor data
for ( auto i : m_dDataPtrAttrs )
for ( auto i : dSpecials )
{
const BYTE * pData = *(BYTE**)(rhs.m_pDynamic+i);
const BYTE * pData = *( BYTE ** ) ( rhs.m_pDynamic + i );
if ( pData )
{
int nBytes = sphUnpackPtrAttr ( pData, &pData );
*(BYTE**)(pDst->m_pDynamic+i) = sphPackPtrAttr ( pData, nBytes );
*( BYTE ** ) ( pDst->m_pDynamic + i ) = sphPackPtrAttr ( pData, nBytes );
}
}
}


//////////////////////////////////////////////////////////////////////////

CSphSchema::CSphSchema ( const char * sName )
: m_sName ( sName )
, m_iFirstFieldLenAttr ( -1 )
, m_iLastFieldLenAttr ( -1 )
{
for ( int i=0; i<BUCKET_COUNT; i++ )
m_dBuckets[i] = 0xffff;
for ( auto & dBucket : m_dBuckets )
dBucket = 0xffff;
}


@@ -6237,10 +6255,9 @@ void CSphSchema::Reset ()

m_dFields.Reset();
m_dAttrs.Reset();
for ( int i=0; i<BUCKET_COUNT; i++ )
m_dBuckets[i] = 0xffff;
for ( auto & dBucket : m_dBuckets )
dBucket = 0xffff;
m_dStaticUsed.Reset();
m_dDynamicUsed.Reset();
}


@@ -6412,18 +6429,12 @@ void CSphSchema::SwapAttrs ( CSphVector<CSphColumnInfo> & dAttrs )

//////////////////////////////////////////////////////////////////////////

CSphRsetSchema::CSphRsetSchema()
: m_pIndexSchema ( NULL )
{}


void CSphRsetSchema::Reset ()
{
CSphSchemaHelper::Reset();

m_pIndexSchema = NULL;
m_pIndexSchema = nullptr;
m_dExtraAttrs.Reset();
m_dDynamicUsed.Reset();
}


@@ -6445,9 +6456,7 @@ int CSphRsetSchema::GetRowSize() const
// we copy over dynamic map in case index schema has dynamic attributes
// (that happens in case of inline attributes, or RAM segments in RT indexes)
// so there is no need to add GetDynamicSize() here
return m_pIndexSchema
? m_dDynamicUsed.GetLength() + m_pIndexSchema->GetStaticSize()
: m_dDynamicUsed.GetLength();
return GetDynamicSize () + m_pIndexSchema ? m_pIndexSchema->GetStaticSize() : 0;
}


@@ -6458,13 +6467,6 @@ int CSphRsetSchema::GetStaticSize() const
}


int CSphRsetSchema::GetDynamicSize() const
{
// we copy over dynamic map in case index schema has dynamic attributes
return m_dDynamicUsed.GetLength();
}


int CSphRsetSchema::GetAttrsCount() const
{
return m_dExtraAttrs.GetLength () + ActualLen();
@@ -14580,9 +14582,8 @@ void CSphIndex_VLN::CopyDocinfo ( const CSphQueryContext * pCtx, CSphMatch & tMa

static inline void CalcContextItems ( CSphMatch & tMatch, const CSphVector<CSphQueryContext::CalcItem_t> & dItems )
{
ARRAY_FOREACH ( i, dItems )
for ( const CSphQueryContext::CalcItem_t & tCalc : dItems )
{
const CSphQueryContext::CalcItem_t & tCalc = dItems[i];
switch ( tCalc.m_eType )
{
case SPH_ATTR_INTEGER:
@@ -1552,6 +1552,10 @@ class ISphSchema
/// simple copy; clones either the entire dynamic part, or a part thereof
virtual void CloneMatch ( CSphMatch * pDst, const CSphMatch & rhs ) const = 0;

// clone all raw attrs and only specified dynamics
virtual void CloneMatchSpecial ( CSphMatch * pDst, const CSphMatch &rhs,
const CSphVector<int> &dSpecials ) const = 0;

virtual void SwapAttrs ( CSphVector<CSphColumnInfo> & dAttrs ) = 0;
};

@@ -1562,16 +1566,28 @@ class CSphSchemaHelper : public ISphSchema
public:
void FreeDataPtrs ( CSphMatch * pMatch ) const final;
void CloneMatch ( CSphMatch * pDst, const CSphMatch & rhs ) const final;
void DiscardPtr ( int iAttr );
void CloneMatchSpecial ( CSphMatch * pDst, const CSphMatch &rhs, const CSphVector<int> &dSpecials ) const final;

// exclude vec of rowitems from dataPtrAttrs and return diff back
void SubsetPtrs ( CSphVector<int> &dSpecials );

/// get dynamic row part size
int GetDynamicSize () const final { return m_dDynamicUsed.GetLength (); }

protected:
CSphVector<int> m_dDataPtrAttrs; // rowitems of pointers to data that are stored inside matches
CSphVector<int> m_dDataPtrAttrs; ///< rowitems of pointers to data that are stored inside matches
CSphVector<int> m_dDynamicUsed; ///< dynamic row part map

/// generic InsertAttr() implementation that tracks data ptr attributes
void InsertAttr ( CSphVector<CSphColumnInfo> & dAttrs, CSphVector<int> & dUsed, int iPos, const CSphColumnInfo & tCol, bool bDynamic );
void Reset();

void CopyPtrs ( CSphMatch * pDst, const CSphMatch & rhs ) const;

public:
// free/copy by specified vec of rowitems, assumed to be from SubsetPtrs() call.
static void FreeDataSpecial ( CSphMatch * pMatch, const CSphVector<int> &dSpecials );
static void CopyDataSpecial ( CSphMatch * pDst, const CSphMatch &rhs, const CSphVector<int> &dSpecials );
};


@@ -1620,14 +1636,11 @@ class CSphSchema : public CSphSchemaHelper
void Reset ();

/// get row size (static+dynamic combined)
virtual int GetRowSize () const { return m_dStaticUsed.GetLength() + m_dDynamicUsed.GetLength(); }
virtual int GetRowSize () const { return GetStaticSize () + GetDynamicSize(); }

/// get static row part size
virtual int GetStaticSize () const { return m_dStaticUsed.GetLength(); }

/// get dynamic row part size
virtual int GetDynamicSize () const { return m_dDynamicUsed.GetLength(); }

virtual int GetAttrsCount () const { return m_dAttrs.GetLength(); }
virtual int GetFieldsCount() const { return m_dFields.GetLength(); }

@@ -1680,7 +1693,6 @@ class CSphSchema : public CSphSchemaHelper
CSphVector<CSphColumnInfo> m_dFields; ///< my fulltext-searchable fields
CSphVector<CSphColumnInfo> m_dAttrs; ///< all my attributes
CSphVector<int> m_dStaticUsed; ///< static row part map (amount of used bits in each rowitem)
CSphVector<int> m_dDynamicUsed; ///< dynamic row part map


/// returns 0xffff if bucket list is empty and position otherwise
@@ -1703,20 +1715,16 @@ class CSphSchema : public CSphSchemaHelper
class CSphRsetSchema : public CSphSchemaHelper
{
protected:
const CSphSchema * m_pIndexSchema; ///< original index schema, for the static part
const CSphSchema * m_pIndexSchema = nullptr; ///< original index schema, for the static part
CSphVector<CSphColumnInfo> m_dExtraAttrs; ///< additional dynamic attributes, for the dynamic one
CSphVector<int> m_dDynamicUsed; ///< dynamic row part map
CSphVector<int> m_dRemoved; ///< original indexes that are suppressed from the index schema by RemoveStaticAttr()

private:
int ActualLen() const; ///< len of m_pIndexSchema accounting removed stuff

public:
CSphRsetSchema();
~CSphRsetSchema() override
{
assert (true);
};
~CSphRsetSchema() override {}

CSphRsetSchema & operator = ( const ISphSchema & rhs );
CSphRsetSchema & operator = ( const CSphSchema & rhs );

@@ -1727,7 +1735,6 @@ class CSphRsetSchema : public CSphSchemaHelper
public:
virtual int GetRowSize() const;
virtual int GetStaticSize() const;
virtual int GetDynamicSize() const;
virtual int GetAttrsCount() const;
virtual int GetFieldsCount() const;
virtual int GetAttrIndex ( const char * sName ) const;
Loading

0 comments on commit 56fdbc9

Please sign in to comment.