Skip to content

Commit

Permalink
Merge pull request #3482 from mhugent/joins_over_several_tables2
Browse files Browse the repository at this point in the history 
Joins over several tables2
  • Loading branch information
@mhugent
mhugent committed Sep 13, 2016
2 parents c151724 + bb4d83f commit 5a4a67a
Show file tree
Hide file tree
Showing 3 changed files with 301 additions and 103 deletions.
287 changes: 187 additions & 100 deletions src/core/qgsvectorlayerfeatureiterator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,7 @@ QgsVectorLayerFeatureIterator::QgsVectorLayerFeatureIterator( QgsVectorLayerFeat
, mFetchedFid( false )
, mEditGeometrySimplifier( nullptr )
{
prepareExpressions();

// prepare joins: may add more attributes to fetch (in order to allow join)
if ( mSource->mJoinBuffer->containsJoins() )
prepareJoins();
prepareFields();

mHasVirtualAttributes = !mFetchJoinInfo.isEmpty() || !mExpressionFieldInfo.isEmpty();

Expand Down Expand Up @@ -472,147 +468,214 @@ void QgsVectorLayerFeatureIterator::rewindEditBuffer()
mFetchChangedGeomIt = mSource->mChangedGeometries.constBegin();
}

void QgsVectorLayerFeatureIterator::prepareJoin( int fieldIdx )
{
if ( !mSource->mFields.exists( fieldIdx ) )
return;

if ( mSource->mFields.fieldOrigin( fieldIdx ) != QgsFields::OriginJoin )
return;

void QgsVectorLayerFeatureIterator::prepareJoins()
{
QgsAttributeList fetchAttributes = ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes ) ? mRequest.subsetOfAttributes() : mSource->mFields.allAttributesList();
QgsAttributeList sourceJoinFields; // attributes that also need to be fetched from this layer in order to have joins working
int sourceLayerIndex;
const QgsVectorJoinInfo* joinInfo = mSource->mJoinBuffer->joinForFieldIndex( fieldIdx, mSource->mFields, sourceLayerIndex );
Q_ASSERT( joinInfo );

mFetchJoinInfo.clear();
QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo->joinLayerId ) );
Q_ASSERT( joinLayer );

for ( QgsAttributeList::const_iterator attIt = fetchAttributes.constBegin(); attIt != fetchAttributes.constEnd(); ++attIt )
if ( !mFetchJoinInfo.contains( joinInfo ) )
{
if ( !mSource->mFields.exists( *attIt ) )
continue;
FetchJoinInfo info;
info.joinInfo = joinInfo;
info.joinLayer = joinLayer;
info.indexOffset = mSource->mJoinBuffer->joinedFieldsOffset( joinInfo, mSource->mFields );

if ( mSource->mFields.fieldOrigin( *attIt ) != QgsFields::OriginJoin )
continue;
if ( joinInfo->targetFieldName.isEmpty() )
info.targetField = joinInfo->targetFieldIndex; //for compatibility with 1.x
else
info.targetField = mSource->mFields.indexFromName( joinInfo->targetFieldName );

if ( joinInfo->joinFieldName.isEmpty() )
info.joinField = joinInfo->joinFieldIndex; //for compatibility with 1.x
else
info.joinField = joinLayer->fields().indexFromName( joinInfo->joinFieldName );

// for joined fields, we always need to request the targetField from the provider too
if ( !mPreparedFields.contains( info.targetField ) && !mFieldsToPrepare.contains( info.targetField ) )
mFieldsToPrepare << info.targetField;

if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes && !mRequest.subsetOfAttributes().contains( info.targetField ) )
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << info.targetField );

mFetchJoinInfo.insert( joinInfo, info );
}

// store field source index - we'll need it when fetching from provider
mFetchJoinInfo[ joinInfo ].attributes.push_back( sourceLayerIndex );
}

void QgsVectorLayerFeatureIterator::prepareExpression( int fieldIdx )
{
const QList<QgsExpressionFieldBuffer::ExpressionField>& exps = mSource->mExpressionFieldBuffer->expressions();

int oi = mSource->mFields.fieldOriginIndex( fieldIdx );
QgsExpression* exp = new QgsExpression( exps[oi].cachedExpression );

int sourceLayerIndex;
const QgsVectorJoinInfo* joinInfo = mSource->mJoinBuffer->joinForFieldIndex( *attIt, mSource->mFields, sourceLayerIndex );
Q_ASSERT( joinInfo );
QgsDistanceArea da;
da.setSourceCrs( mSource->mCrsId );
da.setEllipsoidalMode( true );
da.setEllipsoid( QgsProject::instance()->readEntry( "Measure", "/Ellipsoid", GEO_NONE ) );
exp->setGeomCalculator( da );
exp->setDistanceUnits( QgsProject::instance()->distanceUnits() );
exp->setAreaUnits( QgsProject::instance()->areaUnits() );

QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo->joinLayerId ) );
Q_ASSERT( joinLayer );
exp->prepare( mExpressionContext.data() );
mExpressionFieldInfo.insert( fieldIdx, exp );

if ( !mFetchJoinInfo.contains( joinInfo ) )
Q_FOREACH ( const QString& col, exp->referencedColumns() )
{
int dependantFieldIdx = mSource->mFields.fieldNameIndex( col );
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
{
FetchJoinInfo info;
info.joinInfo = joinInfo;
info.joinLayer = joinLayer;
info.indexOffset = mSource->mJoinBuffer->joinedFieldsOffset( joinInfo, mSource->mFields );

if ( joinInfo->targetFieldName.isEmpty() )
info.targetField = joinInfo->targetFieldIndex; //for compatibility with 1.x
else
info.targetField = mSource->mFields.indexFromName( joinInfo->targetFieldName );

if ( joinInfo->joinFieldName.isEmpty() )
info.joinField = joinInfo->joinFieldIndex; //for compatibility with 1.x
else
info.joinField = joinLayer->fields().indexFromName( joinInfo->joinFieldName );

// for joined fields, we always need to request the targetField from the provider too
if ( !fetchAttributes.contains( info.targetField ) )
sourceJoinFields << info.targetField;

mFetchJoinInfo.insert( joinInfo, info );
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << dependantFieldIdx );
}

// store field source index - we'll need it when fetching from provider
mFetchJoinInfo[ joinInfo ].attributes.push_back( sourceLayerIndex );
// also need to fetch this dependant field
if ( !mPreparedFields.contains( dependantFieldIdx ) && !mFieldsToPrepare.contains( dependantFieldIdx ) )
mFieldsToPrepare << dependantFieldIdx;
}

// add sourceJoinFields if we're using a subset
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() + sourceJoinFields );
if ( exp->needsGeometry() )
{
mRequest.setFlags( mRequest.flags() & ~QgsFeatureRequest::NoGeometry );
}
}

void QgsVectorLayerFeatureIterator::prepareExpressions()
void QgsVectorLayerFeatureIterator::prepareFields()
{
const QList<QgsExpressionFieldBuffer::ExpressionField> exps = mSource->mExpressionFieldBuffer->expressions();
mPreparedFields.clear();
mFieldsToPrepare.clear();
mFetchJoinInfo.clear();
mOrderedJoinInfoList.clear();

mExpressionContext.reset( new QgsExpressionContext() );
mExpressionContext->appendScope( QgsExpressionContextUtils::globalScope() );
mExpressionContext->appendScope( QgsExpressionContextUtils::projectScope() );
mExpressionContext->setFields( mSource->mFields );

QList< int > virtualFieldsToFetch;
for ( int i = 0; i < mSource->mFields.count(); i++ )
mFieldsToPrepare = ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes ) ? mRequest.subsetOfAttributes() : mSource->mFields.allAttributesList();

while ( !mFieldsToPrepare.isEmpty() )
{
if ( mSource->mFields.fieldOrigin( i ) == QgsFields::OriginExpression )
{
// Only prepare if there is no subset defined or the subset contains this field
if ( !( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
|| mRequest.subsetOfAttributes().contains( i ) )
{
virtualFieldsToFetch << i;
}
}
int fieldIdx = mFieldsToPrepare.takeFirst();
if ( mPreparedFields.contains( fieldIdx ) )
continue;

mPreparedFields << fieldIdx;
prepareField( fieldIdx );
}

QList< int > virtualFieldsProcessed;
while ( !virtualFieldsToFetch.isEmpty() )
//sort joins by dependency
if ( mFetchJoinInfo.size() > 0 )
{
int fieldIdx = virtualFieldsToFetch.takeFirst();
if ( virtualFieldsProcessed.contains( fieldIdx ) )
continue;
createOrderedJoinList();
}
}

virtualFieldsProcessed << fieldIdx;
void QgsVectorLayerFeatureIterator::createOrderedJoinList()
{
mOrderedJoinInfoList = mFetchJoinInfo.values();
if ( mOrderedJoinInfoList.size() < 2 )
{
return;
}

QSet<int> resolvedFields; //todo: get provider / virtual fields without joins

int oi = mSource->mFields.fieldOriginIndex( fieldIdx );
QgsExpression* exp = new QgsExpression( exps[oi].cachedExpression );
//add all provider fields without joins as resolved fields
QList< int >::const_iterator prepFieldIt = mPreparedFields.constBegin();
for ( ; prepFieldIt != mPreparedFields.constEnd(); ++prepFieldIt )
{
if ( mSource->mFields.fieldOrigin( *prepFieldIt ) != QgsFields::OriginJoin )
{
resolvedFields.insert( *prepFieldIt );
}
}

QgsDistanceArea da;
da.setSourceCrs( mSource->mCrsId );
da.setEllipsoidalMode( true );
da.setEllipsoid( QgsProject::instance()->readEntry( "Measure", "/Ellipsoid", GEO_NONE ) );
exp->setGeomCalculator( da );
exp->setDistanceUnits( QgsProject::instance()->distanceUnits() );
exp->setAreaUnits( QgsProject::instance()->areaUnits() );
//iterate through the joins. If target field is not yet covered, move the entry to the end of the list

exp->prepare( mExpressionContext.data() );
mExpressionFieldInfo.insert( fieldIdx, exp );
//some join combinations might not have a resolution at all
int maxIterations = ( mOrderedJoinInfoList.size() + 1 ) * mOrderedJoinInfoList.size() / 2.0;
int currentIteration = 0;

Q_FOREACH ( const QString& col, exp->referencedColumns() )
for ( int i = 0; i < mOrderedJoinInfoList.size() - 1; ++i )
{
if ( !resolvedFields.contains( mOrderedJoinInfoList.at( i ).targetField ) )
{
mOrderedJoinInfoList.append( mOrderedJoinInfoList.at( i ) );
mOrderedJoinInfoList.removeAt( i );
--i;
}
else
{
int dependantFieldIdx = mSource->mFields.fieldNameIndex( col );
if ( mRequest.flags() & QgsFeatureRequest::SubsetOfAttributes )
int offset = mOrderedJoinInfoList.at( i ).indexOffset;
int joinField = mOrderedJoinInfoList.at( i ).joinField;

QgsAttributeList attributes = mOrderedJoinInfoList.at( i ).attributes;
QgsAttributeList::const_iterator attIt = attributes.constBegin();
for ( ; attIt != attributes.constEnd(); ++attIt )
{
mRequest.setSubsetOfAttributes( mRequest.subsetOfAttributes() << dependantFieldIdx );
if ( *attIt != joinField )
{
resolvedFields.insert( joinField < *attIt ? *attIt + offset - 1 : *attIt + offset );
}
}
// also need to fetch this dependant field
if ( mSource->mFields.fieldOrigin( dependantFieldIdx ) == QgsFields::OriginExpression )
virtualFieldsToFetch << dependantFieldIdx;
}

if ( exp->needsGeometry() )
++currentIteration;
if ( currentIteration >= maxIterations )
{
mRequest.setFlags( mRequest.flags() & ~QgsFeatureRequest::NoGeometry );
break;
}
}
}

void QgsVectorLayerFeatureIterator::prepareField( int fieldIdx )
{
switch ( mSource->mFields.fieldOrigin( fieldIdx ) )
{
case QgsFields::OriginExpression:
prepareExpression( fieldIdx );
break;

case QgsFields::OriginJoin:
if ( mSource->mJoinBuffer->containsJoins() )
{
prepareJoin( fieldIdx );
}
break;

case QgsFields::OriginUnknown:
case QgsFields::OriginProvider:
case QgsFields::OriginEdit:
break;
}
}

void QgsVectorLayerFeatureIterator::addJoinedAttributes( QgsFeature &f )
{
QMap<const QgsVectorJoinInfo*, FetchJoinInfo>::const_iterator joinIt = mFetchJoinInfo.constBegin();
for ( ; joinIt != mFetchJoinInfo.constEnd(); ++joinIt )
QList< FetchJoinInfo >::const_iterator joinIt = mOrderedJoinInfoList.constBegin();
for ( ; joinIt != mOrderedJoinInfoList.constEnd(); ++joinIt )
{
const FetchJoinInfo& info = joinIt.value();
Q_ASSERT( joinIt.key() );
QVariant targetFieldValue = f.attribute( joinIt->targetField );

QVariant targetFieldValue = f.attribute( info.targetField );
if ( !targetFieldValue.isValid() )
continue;

const QHash< QString, QgsAttributes>& memoryCache = info.joinInfo->cachedAttributes;
const QHash< QString, QgsAttributes>& memoryCache = joinIt->joinInfo->cachedAttributes;
if ( memoryCache.isEmpty() )
info.addJoinedAttributesDirect( f, targetFieldValue );
joinIt->addJoinedAttributesDirect( f, targetFieldValue );
else
info.addJoinedAttributesCached( f, targetFieldValue );
joinIt->addJoinedAttributesCached( f, targetFieldValue );
}
}

Expand All @@ -623,24 +686,48 @@ void QgsVectorLayerFeatureIterator::addVirtualAttributes( QgsFeature& f )
attr.resize( mSource->mFields.count() ); // Provider attrs count + joined attrs count + expression attrs count
f.setAttributes( attr );

// possible TODO - handle combinations of expression -> join -> expression -> join?
// but for now, write that off as too complex and an unlikely rare, unsupported use case

QList< int > fetchedVirtualAttributes;
//first, check through joins for any virtual fields we need
QMap<const QgsVectorJoinInfo*, FetchJoinInfo>::const_iterator joinIt = mFetchJoinInfo.constBegin();
for ( ; joinIt != mFetchJoinInfo.constEnd(); ++joinIt )
{
if ( mExpressionFieldInfo.contains( joinIt->targetField ) )
{
// have to calculate expression field before we can handle this join
addExpressionAttribute( f, joinIt->targetField );
fetchedVirtualAttributes << joinIt->targetField;
}
}

if ( !mFetchJoinInfo.isEmpty() )
addJoinedAttributes( f );

// add remaining expression fields
if ( !mExpressionFieldInfo.isEmpty() )
{
QMap<int, QgsExpression*>::ConstIterator it = mExpressionFieldInfo.constBegin();

for ( ; it != mExpressionFieldInfo.constEnd(); ++it )
{
QgsExpression* exp = it.value();
mExpressionContext->setFeature( f );
QVariant val = exp->evaluate( mExpressionContext.data() );
mSource->mFields.at( it.key() ).convertCompatible( val );
f.setAttribute( it.key(), val );
if ( fetchedVirtualAttributes.contains( it.key() ) )
continue;

addExpressionAttribute( f, it.key() );
}
}
}

void QgsVectorLayerFeatureIterator::addExpressionAttribute( QgsFeature& f, int attrIndex )
{
QgsExpression* exp = mExpressionFieldInfo.value( attrIndex );
mExpressionContext->setFeature( f );
QVariant val = exp->evaluate( mExpressionContext.data() );
mSource->mFields.at( attrIndex ).convertCompatible( val );
f.setAttribute( attrIndex, val );
}

bool QgsVectorLayerFeatureIterator::prepareSimplification( const QgsSimplifyMethod& simplifyMethod )
{
delete mEditGeometrySimplifier;
Expand Down
Loading

0 comments on commit 5a4a67a

Please sign in to comment.