Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Joins over several tables2 #3482

Merged
merged 3 commits into from Sep 13, 2016
Merged

Joins over several tables2 #3482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c27c9b5
Correctly support joins using virtual fields
nyalldawson Jun 10, 2016
45e485c
Fix joins over several tables
mhugent Sep 12, 2016
bb4d83f
Fix imports in test_qgsfeatureiterator
mhugent Sep 13, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
287 changes: 187 additions & 100 deletions src/core/qgsvectorlayerfeatureiterator.cpp
View file
Open in desktop
Expand Up @@ -95,11 +95,7 @@ QgsVectorLayerFeatureIterator::QgsVectorLayerFeatureIterator( QgsVectorLayerFeat
, mFetchedFid( false )
, mEditGeometrySimplifier( nullptr )
{
prepareExpressions();

// prepare joins: may add more attributes to fetch (in order to allow join)
if ( mSource->mJoinBuffer->containsJoins() )
prepareJoins();
prepareFields();

mHasVirtualAttributes = !mFetchJoinInfo.isEmpty() || !mExpressionFieldInfo.isEmpty();

Expand Down Expand Up @@ -472,147 +468,214 @@ void QgsVectorLayerFeatureIterator::rewindEditBuffer()
mFetchChangedGeomIt = mSource->mChangedGeometries.constBegin();
}

void QgsVectorLayerFeatureIterator::prepareJoin( int fieldIdx )
{
if ( !mSource->mFields.exists( fieldIdx ) )
return;

if ( mSource->mFields.fieldOrigin( fieldIdx ) != QgsFields::OriginJoin )
return;

void QgsVectorLayerFeatureIterator::prepareJoins()
{
QgsAttributeList fetchAttributes = ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes ) ? mRequest.subsetOfAttributes() : mSource->mFields.allAttributesList();
QgsAttributeList sourceJoinFields; // attributes that also need to be fetched from this layer in order to have joins working
int sourceLayerIndex;
const QgsVectorJoinInfo* joinInfo = mSource->mJoinBuffer->joinForFieldIndex( fieldIdx, mSource->mFields, sourceLayerIndex );
Q_ASSERT( joinInfo );

mFetchJoinInfo.clear();
QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo->joinLayerId ) );
Q_ASSERT( joinLayer );

for ( QgsAttributeList::const_iterator attIt = fetchAttributes.constBegin(); attIt != fetchAttributes.constEnd(); ++attIt )
if ( !mFetchJoinInfo.contains( joinInfo ) )
{
if ( !mSource->mFields.exists( *attIt ) )
continue;
FetchJoinInfo info;
info.joinInfo = joinInfo;
info.joinLayer = joinLayer;
info.indexOffset = mSource->mJoinBuffer->joinedFieldsOffset( joinInfo, mSource->mFields );

if ( mSource->mFields.fieldOrigin( *attIt ) != QgsFields::OriginJoin )
continue;
if ( joinInfo->targetFieldName.isEmpty() )
info.targetField = joinInfo->targetFieldIndex; //for compatibility with 1.x
else
info.targetField = mSource->mFields.indexFromName( joinInfo->targetFieldName );

if ( joinInfo->joinFieldName.isEmpty() )
info.joinField = joinInfo->joinFieldIndex; //for compatibility with 1.x
else
info.joinField = joinLayer->fields().indexFromName( joinInfo->joinFieldName );

// for joined fields, we always need to request the targetField from the provider too
if ( !mPreparedFields.contains( info.targetField ) && !mFieldsToPrepare.contains( info.targetField ) )
mFieldsToPrepare << info.targetField;

if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes && !mRequest.subsetOfAttributes().contains( info.targetField ) )
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << info.targetField );

mFetchJoinInfo.insert( joinInfo, info );
}

// store field source index - we'll need it when fetching from provider
mFetchJoinInfo[ joinInfo ].attributes.push_back( sourceLayerIndex );
}

void QgsVectorLayerFeatureIterator::prepareExpression( int fieldIdx )
{
const QList<QgsExpressionFieldBuffer::ExpressionField>& exps = mSource->mExpressionFieldBuffer->expressions();

int oi = mSource->mFields.fieldOriginIndex( fieldIdx );
QgsExpression* exp = new QgsExpression( exps[oi].cachedExpression );

int sourceLayerIndex;
const QgsVectorJoinInfo* joinInfo = mSource->mJoinBuffer->joinForFieldIndex( *attIt, mSource->mFields, sourceLayerIndex );
Q_ASSERT( joinInfo );
QgsDistanceArea da;
da.setSourceCrs( mSource->mCrsId );
da.setEllipsoidalMode( true );
da.setEllipsoid( QgsProject::instance()->readEntry( "Measure", "/Ellipsoid", GEO_NONE ) );
exp->setGeomCalculator( da );
exp->setDistanceUnits( QgsProject::instance()->distanceUnits() );
exp->setAreaUnits( QgsProject::instance()->areaUnits() );

QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo->joinLayerId ) );
Q_ASSERT( joinLayer );
exp->prepare( mExpressionContext.data() );
mExpressionFieldInfo.insert( fieldIdx, exp );

if ( !mFetchJoinInfo.contains( joinInfo ) )
Q_FOREACH ( const QString& col, exp->referencedColumns() )
{
int dependantFieldIdx = mSource->mFields.fieldNameIndex( col );
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
FetchJoinInfo info;
info.joinInfo = joinInfo;
info.joinLayer = joinLayer;
info.indexOffset = mSource->mJoinBuffer->joinedFieldsOffset( joinInfo, mSource->mFields );

if ( joinInfo->targetFieldName.isEmpty() )
info.targetField = joinInfo->targetFieldIndex; //for compatibility with 1.x
else
info.targetField = mSource->mFields.indexFromName( joinInfo->targetFieldName );

if ( joinInfo->joinFieldName.isEmpty() )
info.joinField = joinInfo->joinFieldIndex; //for compatibility with 1.x
else
info.joinField = joinLayer->fields().indexFromName( joinInfo->joinFieldName );

// for joined fields, we always need to request the targetField from the provider too
if ( !fetchAttributes.contains( info.targetField ) )
sourceJoinFields << info.targetField;

mFetchJoinInfo.insert( joinInfo, info );
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << dependantFieldIdx );
}

// store field source index - we'll need it when fetching from provider
mFetchJoinInfo[ joinInfo ].attributes.push_back( sourceLayerIndex );
// also need to fetch this dependant field
if ( !mPreparedFields.contains( dependantFieldIdx ) && !mFieldsToPrepare.contains( dependantFieldIdx ) )
mFieldsToPrepare << dependantFieldIdx;
}

// add sourceJoinFields if we're using a subset
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() + sourceJoinFields );
if ( exp->needsGeometry() )
{
mRequest.setFlags( mRequest.flags() & ~QgsFeatureRequest::NoGeometry );
}
}

void QgsVectorLayerFeatureIterator::prepareExpressions()
void QgsVectorLayerFeatureIterator::prepareFields()
{
const QList<QgsExpressionFieldBuffer::ExpressionField> exps = mSource->mExpressionFieldBuffer->expressions();
mPreparedFields.clear();
mFieldsToPrepare.clear();
mFetchJoinInfo.clear();
mOrderedJoinInfoList.clear();

mExpressionContext.reset( new QgsExpressionContext() );
mExpressionContext->appendScope( QgsExpressionContextUtils::globalScope() );
mExpressionContext->appendScope( QgsExpressionContextUtils::projectScope() );
mExpressionContext->setFields( mSource->mFields );

QList< int > virtualFieldsToFetch;
for ( int i = 0; i < mSource->mFields.count(); i++ )
mFieldsToPrepare = ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes ) ? mRequest.subsetOfAttributes() : mSource->mFields.allAttributesList();

while ( !mFieldsToPrepare.isEmpty() )
{
if ( mSource->mFields.fieldOrigin( i ) == QgsFields::OriginExpression )
{
// Only prepare if there is no subset defined or the subset contains this field
if ( !( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
|| mRequest.subsetOfAttributes().contains( i ) )
{
virtualFieldsToFetch << i;
}
}
int fieldIdx = mFieldsToPrepare.takeFirst();
if ( mPreparedFields.contains( fieldIdx ) )
continue;

mPreparedFields << fieldIdx;
prepareField( fieldIdx );
}

QList< int > virtualFieldsProcessed;
while ( !virtualFieldsToFetch.isEmpty() )
//sort joins by dependency
if ( mFetchJoinInfo.size() > 0 )
{
int fieldIdx = virtualFieldsToFetch.takeFirst();
if ( virtualFieldsProcessed.contains( fieldIdx ) )
continue;
createOrderedJoinList();
}
}

virtualFieldsProcessed << fieldIdx;
void QgsVectorLayerFeatureIterator::createOrderedJoinList()
{
mOrderedJoinInfoList = mFetchJoinInfo.values();
if ( mOrderedJoinInfoList.size() < 2 )
{
return;
}

QSet<int> resolvedFields; //todo: get provider / virtual fields without joins

int oi = mSource->mFields.fieldOriginIndex( fieldIdx );
QgsExpression* exp = new QgsExpression( exps[oi].cachedExpression );
//add all provider fields without joins as resolved fields
QList< int >::const_iterator prepFieldIt = mPreparedFields.constBegin();
for ( ; prepFieldIt != mPreparedFields.constEnd(); ++prepFieldIt )
{
if ( mSource->mFields.fieldOrigin( *prepFieldIt ) != QgsFields::OriginJoin )
{
resolvedFields.insert( *prepFieldIt );
}
}

QgsDistanceArea da;
da.setSourceCrs( mSource->mCrsId );
da.setEllipsoidalMode( true );
da.setEllipsoid( QgsProject::instance()->readEntry( "Measure", "/Ellipsoid", GEO_NONE ) );
exp->setGeomCalculator( da );
exp->setDistanceUnits( QgsProject::instance()->distanceUnits() );
exp->setAreaUnits( QgsProject::instance()->areaUnits() );
//iterate through the joins. If target field is not yet covered, move the entry to the end of the list

exp->prepare( mExpressionContext.data() );
mExpressionFieldInfo.insert( fieldIdx, exp );
//some join combinations might not have a resolution at all
int maxIterations = ( mOrderedJoinInfoList.size() + 1 ) * mOrderedJoinInfoList.size() / 2.0;
int currentIteration = 0;

Q_FOREACH ( const QString& col, exp->referencedColumns() )
for ( int i = 0; i < mOrderedJoinInfoList.size() - 1; ++i )
{
if ( !resolvedFields.contains( mOrderedJoinInfoList.at( i ).targetField ) )
{
mOrderedJoinInfoList.append( mOrderedJoinInfoList.at( i ) );
mOrderedJoinInfoList.removeAt( i );
--i;
}
else
{
int dependantFieldIdx = mSource->mFields.fieldNameIndex( col );
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
int offset = mOrderedJoinInfoList.at( i ).indexOffset;
int joinField = mOrderedJoinInfoList.at( i ).joinField;

QgsAttributeList attributes = mOrderedJoinInfoList.at( i ).attributes;
QgsAttributeList::const_iterator attIt = attributes.constBegin();
for ( ; attIt != attributes.constEnd(); ++attIt )
{
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << dependantFieldIdx );
if ( *attIt != joinField )
{
resolvedFields.insert( joinField < *attIt ? *attIt + offset - 1 : *attIt + offset );
}
}
// also need to fetch this dependant field
if ( mSource->mFields.fieldOrigin( dependantFieldIdx ) == QgsFields::OriginExpression )
virtualFieldsToFetch << dependantFieldIdx;
}

if ( exp->needsGeometry() )
++currentIteration;
if ( currentIteration >= maxIterations )
{
mRequest.setFlags( mRequest.flags() & ~QgsFeatureRequest::NoGeometry );
break;
}
}
}

void QgsVectorLayerFeatureIterator::prepareField( int fieldIdx )
{
switch ( mSource->mFields.fieldOrigin( fieldIdx ) )
{
case QgsFields::OriginExpression:
prepareExpression( fieldIdx );
break;

case QgsFields::OriginJoin:
if ( mSource->mJoinBuffer->containsJoins() )
{
prepareJoin( fieldIdx );
}
break;

case QgsFields::OriginUnknown:
case QgsFields::OriginProvider:
case QgsFields::OriginEdit:
break;
}
}

void QgsVectorLayerFeatureIterator::addJoinedAttributes( QgsFeature &f )
{
QMap<const QgsVectorJoinInfo*, FetchJoinInfo>::const_iterator joinIt = mFetchJoinInfo.constBegin();
for ( ; joinIt != mFetchJoinInfo.constEnd(); ++joinIt )
QList< FetchJoinInfo >::const_iterator joinIt = mOrderedJoinInfoList.constBegin();
for ( ; joinIt != mOrderedJoinInfoList.constEnd(); ++joinIt )
{
const FetchJoinInfo& info = joinIt.value();
Q_ASSERT( joinIt.key() );
QVariant targetFieldValue = f.attribute( joinIt->targetField );

QVariant targetFieldValue = f.attribute( info.targetField );
if ( !targetFieldValue.isValid() )
continue;

const QHash< QString, QgsAttributes>& memoryCache = info.joinInfo->cachedAttributes;
const QHash< QString, QgsAttributes>& memoryCache = joinIt->joinInfo->cachedAttributes;
if ( memoryCache.isEmpty() )
info.addJoinedAttributesDirect( f, targetFieldValue );
joinIt->addJoinedAttributesDirect( f, targetFieldValue );
else
info.addJoinedAttributesCached( f, targetFieldValue );
joinIt->addJoinedAttributesCached( f, targetFieldValue );
}
}

Expand All @@ -623,24 +686,48 @@ void QgsVectorLayerFeatureIterator::addVirtualAttributes( QgsFeature& f )
attr.resize( mSource->mFields.count() ); // Provider attrs count + joined attrs count + expression attrs count
f.setAttributes( attr );

// possible TODO - handle combinations of expression -> join -> expression -> join?
// but for now, write that off as too complex and an unlikely rare, unsupported use case

QList< int > fetchedVirtualAttributes;
//first, check through joins for any virtual fields we need
QMap<const QgsVectorJoinInfo*, FetchJoinInfo>::const_iterator joinIt = mFetchJoinInfo.constBegin();
for ( ; joinIt != mFetchJoinInfo.constEnd(); ++joinIt )
{
if ( mExpressionFieldInfo.contains( joinIt->targetField ) )
{
// have to calculate expression field before we can handle this join
addExpressionAttribute( f, joinIt->targetField );
fetchedVirtualAttributes << joinIt->targetField;
}
}

if ( !mFetchJoinInfo.isEmpty() )
addJoinedAttributes( f );

// add remaining expression fields
if ( !mExpressionFieldInfo.isEmpty() )
{
QMap<int, QgsExpression*>::ConstIterator it = mExpressionFieldInfo.constBegin();

for ( ; it != mExpressionFieldInfo.constEnd(); ++it )
{
QgsExpression* exp = it.value();
mExpressionContext->setFeature( f );
QVariant val = exp->evaluate( mExpressionContext.data() );
mSource->mFields.at( it.key() ).convertCompatible( val );
f.setAttribute( it.key(), val );
if ( fetchedVirtualAttributes.contains( it.key() ) )
continue;

addExpressionAttribute( f, it.key() );
}
}
}

void QgsVectorLayerFeatureIterator::addExpressionAttribute( QgsFeature& f, int attrIndex )
{
QgsExpression* exp = mExpressionFieldInfo.value( attrIndex );
mExpressionContext->setFeature( f );
QVariant val = exp->evaluate( mExpressionContext.data() );
mSource->mFields.at( attrIndex ).convertCompatible( val );
f.setAttribute( attrIndex, val );
}

bool QgsVectorLayerFeatureIterator::prepareSimplification( const QgsSimplifyMethod& simplifyMethod )
{
delete mEditGeometrySimplifier;
Expand Down