Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Added possibility to cache join layer in memory. Still experimental code 
git-svn-id: http://svn.osgeo.org/qgis/branches/table_join_branch@14107 c8812cc2-4d05-0410-92ff-de0c093fc19c
  • Loading branch information
mhugent committed Aug 18, 2010
1 parent a3270b9 commit d603091
Show file tree
Hide file tree
Showing 11 changed files with 199 additions and 84 deletions.
1 change: 1 addition & 0 deletions images/images.qrc
View file
Expand Up @@ -257,6 +257,7 @@
<file>themes/default/propertyicons/digitising.png</file>
<file>themes/default/propertyicons/general.png</file>
<file>themes/default/propertyicons/histogram.png</file>
<file>themes/default/propertyicons/join.png</file>
<file>themes/default/propertyicons/labels.png</file>
<file>themes/default/propertyicons/locale.png</file>
<file>themes/default/propertyicons/map_tools.png</file>
Expand Down
Binary file added images/themes/default/propertyicons/join.png
View file
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 7 additions & 1 deletion src/app/qgsaddjoindialog.cpp
View file
Expand Up @@ -58,6 +58,8 @@ QgsAddJoinDialog::QgsAddJoinDialog( QgsVectorLayer* layer, QWidget * parent, Qt:
{
mTargetFieldComboBox->addItem(fieldIt.value().name(), fieldIt.key() );
}

mCacheInMemoryCheckBox->setChecked( true );
}

QgsAddJoinDialog::~QgsAddJoinDialog()
Expand Down Expand Up @@ -89,6 +91,11 @@ QString QgsAddJoinDialog::targetFieldName() const
return mTargetFieldComboBox->itemText( mTargetFieldComboBox->currentIndex() );
}

bool QgsAddJoinDialog::cacheInMemory() const
{
return mCacheInMemoryCheckBox->isChecked();
}

bool QgsAddJoinDialog::createAttributeIndex() const
{
return mCreateIndexCheckBox->isChecked();
Expand Down Expand Up @@ -121,7 +128,6 @@ void QgsAddJoinDialog::on_mJoinLayerComboBox_currentIndexChanged ( int index )
if( dp && (dp->capabilities() & QgsVectorDataProvider::CreateAttributeIndex) )
{
mCreateIndexCheckBox->setEnabled( true );
mCreateIndexCheckBox->setChecked( true );
}
else
{
Expand Down
2 changes: 2 additions & 0 deletions src/app/qgsaddjoindialog.h
View file
Expand Up @@ -40,6 +40,8 @@ class QgsAddJoinDialog: public QDialog, private Ui::QgsAddJoinDialogBase
int targetField() const;
/**Returns the name of the target field (join-to field)*/
QString targetFieldName() const;
/**True if joined layer should be cached in virtual memory*/
bool cacheInMemory() const;
/**Returns true if user wants to create an attribute index on the join field*/
bool createAttributeIndex() const;

Expand Down
2 changes: 1 addition & 1 deletion src/app/qgsvectorlayerproperties.cpp
View file
Expand Up @@ -1190,7 +1190,7 @@ void QgsVectorLayerProperties::on_mButtonAddJoin_clicked()
}
}

layer->addJoin( info );
layer->addJoin( info, d.cacheInMemory() );
loadRows(); //update attribute tab
addJoinToTreeWidget( info );
}
Expand Down
3 changes: 2 additions & 1 deletion src/core/qgsproject.cpp
View file
Expand Up @@ -736,13 +736,14 @@ QPair< bool, QList<QDomNode> > QgsProject::_getMapLayers( QDomDocument const &do
emit layerLoaded( i + 1, nl.count() );
}

//Update field map of layers with joins.
//Update field map of layers with joins and create join caches if necessary
//Needs to be done here once all dependent layers are loaded
QList<QgsVectorLayer*>::iterator vIt = vLayerList.begin();
for(; vIt != vLayerList.end(); ++vIt )
{
if( (*vIt)->vectorJoins().size() > 0 )
{
(*vIt)->createJoinCaches();
(*vIt)->updateFieldMap();
}
}
Expand Down
200 changes: 135 additions & 65 deletions src/core/qgsvectorlayer.cpp
View file
Expand Up @@ -1486,31 +1486,8 @@ void QgsVectorLayer::updateFeatureAttributes( QgsFeature &f, bool all )
continue;
}

//set subset string
QString bkSubsetString = joinLayer->dataProvider()->subsetString(); //provider might already have a subset string
QString subsetString;
if( !bkSubsetString.isEmpty() )
{
subsetString.append(" AND ");
}
subsetString.append( "\"" + joinFieldName + "\"" + " = " + "\"" + targetFieldValue.toString() + "\"" );
joinLayer->dataProvider()->setSubsetString( subsetString, false );

//select (no geometry)
joinLayer->select( joinLayer->pendingAllAttributesList(), QgsRectangle(), false, false );
addJoinedFeatureAttributes( f, *joinIt, joinFieldName, targetFieldValue, joinLayer->pendingAllAttributesList(), index );

//get first feature
QgsFeature fet;
if( joinLayer->nextFeature( fet ) )
{
QgsAttributeMap attMap = fet.attributeMap();
QgsAttributeMap::const_iterator attIt = attMap.constBegin();
for(; attIt != attMap.constEnd(); ++attIt )
{
f.addAttribute( attIt.key() + index, attIt.value() );
}
}
joinLayer->dataProvider()->setSubsetString( bkSubsetString, false );
maxIndex( joinLayer->pendingFields(), currentMaxIndex );
index += ( currentMaxIndex + 1 );
}
Expand All @@ -1526,44 +1503,19 @@ void QgsVectorLayer::updateFeatureAttributes( QgsFeature &f, bool all )
continue;
}

QString joinFieldName = joinLayer->pendingFields().value( joinIt.value().joinInfo.joinField ).name();
QString joinFieldName = joinLayer->pendingFields().value( joinIt.value().joinInfo->joinField ).name();
if( joinFieldName.isEmpty() )
{
continue;
}

QVariant targetFieldValue = f.attributeMap().value( joinIt->joinInfo.targetField );
QVariant targetFieldValue = f.attributeMap().value( joinIt->joinInfo->targetField );
if( !targetFieldValue.isValid() )
{
continue;
}

//set subset string
QString bkSubsetString = joinLayer->dataProvider()->subsetString(); //provider might already have a subset string
QString subsetString;
if( !bkSubsetString.isEmpty() )
{
subsetString.append(" AND ");
}
subsetString.append( "\"" + joinFieldName + "\"" + " = " + "\"" + targetFieldValue.toString() + "\"" );
joinLayer->dataProvider()->setSubsetString( subsetString, false );

//select (no geometry)
joinLayer->select( joinIt.value().attributes, QgsRectangle(), false, false );

//get first feature
QgsFeature fet;
if( joinLayer->nextFeature( fet ) )
{
QgsAttributeMap attMap = fet.attributeMap();
QgsAttributeMap::const_iterator attIt = attMap.constBegin();
for(; attIt != attMap.constEnd(); ++attIt )
{
f.addAttribute( attIt.key() + joinIt.value().indexOffset, attIt.value() );
}
}

joinLayer->dataProvider()->setSubsetString( bkSubsetString, false );
addJoinedFeatureAttributes( f, *(joinIt.value().joinInfo), joinFieldName, targetFieldValue, joinIt.value().attributes, joinIt.value().indexOffset );
}
}
}
Expand Down Expand Up @@ -1592,6 +1544,73 @@ void QgsVectorLayer::updateFeatureAttributes( QgsFeature &f, bool all )
f.changeAttribute( it.key(), QVariant( QString::null ) );
}

void QgsVectorLayer::addJoinedFeatureAttributes( QgsFeature& f, const QgsVectorJoinInfo& joinInfo, const QString& joinFieldName,
const QVariant& joinValue, const QgsAttributeList& attributes, int attributeIndexOffset )
{
const QHash< QString, QgsAttributeMap>& memoryCache = joinInfo.cachedAttributes;
if( !memoryCache.isEmpty() ) //use join memory cache
{
QgsAttributeMap featureAttributes = memoryCache.value( joinValue.toString() );
bool found = !featureAttributes.isEmpty();
QgsAttributeList::const_iterator attIt = attributes.constBegin();
for(; attIt != attributes.constEnd(); ++attIt )
{
if( found )
{
f.addAttribute( *attIt + attributeIndexOffset, featureAttributes.value( *attIt ) );
}
else
{
f.addAttribute( *attIt + attributeIndexOffset, QVariant() );
}
}
}
else //work with subset string
{
QgsVectorLayer* joinLayer = dynamic_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo.joinLayerId ) );
if( !joinLayer )
{
return;
}

//no memory cache, query the joined values by setting substring
QString subsetString = joinLayer->dataProvider()->subsetString(); //provider might already have a subset string
QString bkSubsetString = subsetString;
if( !subsetString.isEmpty() )
{
subsetString.append(" AND ");
}

subsetString.append( "\"" + joinFieldName + "\"" + " = " + "\"" + joinValue.toString() + "\"" );
joinLayer->dataProvider()->setSubsetString( subsetString, false );

//select (no geometry)
joinLayer->select( attributes, QgsRectangle(), false, false );

//get first feature
QgsFeature fet;
if( joinLayer->nextFeature( fet ) )
{
QgsAttributeMap attMap = fet.attributeMap();
QgsAttributeMap::const_iterator attIt = attMap.constBegin();
for(; attIt != attMap.constEnd(); ++attIt )
{
f.addAttribute( attIt.key() + attributeIndexOffset, attIt.value() );
}
}
else //no suitable join feature found, insert invalid variants
{
QgsAttributeList::const_iterator attIt = attributes.constBegin();
for(; attIt != attributes.constEnd(); ++attIt )
{
f.addAttribute( *attIt + attributeIndexOffset, QVariant() );
}
}

joinLayer->dataProvider()->setSubsetString( bkSubsetString, false );
}
}

void QgsVectorLayer::updateFeatureGeometry( QgsFeature &f )
{
if ( mChangedGeometries.contains( f.id() ) )
Expand Down Expand Up @@ -1632,21 +1651,21 @@ void QgsVectorLayer::select( QgsAttributeList attributes, QgsRectangle rect, boo
}
else
{
QgsVectorJoinInfo joinInfo;
int indexOffset;
if( joinForFieldIndex( *it, joinInfo, indexOffset ) )
const QgsVectorJoinInfo* joinInfo= joinForFieldIndex( *it, indexOffset );
if( joinInfo )
{
QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo.joinLayerId ) );
QgsVectorLayer* joinLayer = qobject_cast<QgsVectorLayer*>( QgsMapLayerRegistry::instance()->mapLayer( joinInfo->joinLayerId ) );
if( joinLayer )
{
mFetchJoinInfos[ joinLayer ].joinInfo = joinInfo;
mFetchJoinInfos[ joinLayer].attributes.push_back( *it - indexOffset ); //store provider index
mFetchJoinInfos[ joinLayer ].indexOffset = indexOffset;
//for joined fields, we always need to request the targetField from the provider too
if( !mFetchAttributes.contains( joinInfo.targetField ))
if( !mFetchAttributes.contains( joinInfo->targetField ))
{
targetJoinFieldList << joinInfo.targetField;
mFetchProvAttributes << joinInfo.targetField;
targetJoinFieldList << joinInfo->targetField;
mFetchProvAttributes << joinInfo->targetField;
}
}
}
Expand Down Expand Up @@ -2628,7 +2647,8 @@ bool QgsVectorLayer::readXml( QDomNode & layer_node )
info.joinField = infoElem.attribute("joinField").toInt();
info.joinLayerId = infoElem.attribute("joinLayerId");
info.targetField = infoElem.attribute("targetField").toInt();
addJoin( info );
bool memoryCache = infoElem.attribute("memoryCache").toInt();
addJoin( info, memoryCache );
}
}

Expand Down Expand Up @@ -2787,6 +2807,7 @@ bool QgsVectorLayer::writeXml( QDomNode & layer_node,
joinElem.setAttribute( "targetField", joinIt->targetField );
joinElem.setAttribute( "joinLayerId", joinIt->joinLayerId );
joinElem.setAttribute( "joinField", joinIt->joinField);
joinElem.setAttribute( "memoryCache", !joinIt->cachedAttributes.isEmpty() );
vectorJoinsElem.appendChild( joinElem );
}

Expand Down Expand Up @@ -4746,9 +4767,50 @@ int QgsVectorLayer::fieldNameIndex( const QString& fieldName ) const
return -1;
}

void QgsVectorLayer::addJoin( QgsVectorJoinInfo joinInfo )
void QgsVectorLayer::cacheJoinLayer( QgsVectorJoinInfo& joinInfo )
{
//memory cache not required or already done
if( !joinInfo.memoryCache || joinInfo.cachedAttributes.size() > 0 )
{
return;
}

QgsVectorLayer* cacheLayer = dynamic_cast<QgsVectorLayer*>(QgsMapLayerRegistry::instance()->mapLayer( joinInfo.joinLayerId ) );
if( cacheLayer )
{
joinInfo.cachedAttributes.clear();
cacheLayer->select( cacheLayer->pendingAllAttributesList(), QgsRectangle(), false, false );
QgsFeature f;
while( cacheLayer->nextFeature( f ) )
{
const QgsAttributeMap& map = f.attributeMap();
joinInfo.cachedAttributes.insert( map.value( joinInfo.joinField).toString(), map );
}
}
}

void QgsVectorLayer::addJoin( QgsVectorJoinInfo joinInfo, bool cacheInMemory )
{
joinInfo.memoryCache = cacheInMemory;
mVectorJoins.push_back( joinInfo );

//cache joined layer to virtual memory if specified by user
if( cacheInMemory )
{
cacheJoinLayer( mVectorJoins.last() );
/*QgsVectorLayer* cacheLayer = dynamic_cast<QgsVectorLayer*>(QgsMapLayerRegistry::instance()->mapLayer( joinInfo.joinLayerId ) );
if( cacheLayer )
{
cacheLayer->select( cacheLayer->pendingAllAttributesList(), QgsRectangle(), false, false );
QgsFeature f;
while( cacheLayer->nextFeature( f ) )
{
const QgsAttributeMap& map = f.attributeMap();
mVectorJoins.last().cachedAttributes.insert( map.value( joinInfo.joinField).toString(), map );
}
}*/
}

updateFieldMap();
}

Expand Down Expand Up @@ -4849,6 +4911,15 @@ void QgsVectorLayer::updateFieldMap()
}
}

void QgsVectorLayer::createJoinCaches()
{
QList< QgsVectorJoinInfo >::iterator joinIt = mVectorJoins.begin();
for(; joinIt != mVectorJoins.end(); ++joinIt)
{
cacheJoinLayer( *joinIt);
}
}

void QgsVectorLayer::stopRendererV2( QgsRenderContext& rendererContext, QgsSingleSymbolRendererV2* selRenderer )
{
mRendererV2->stopRender( rendererContext );
Expand Down Expand Up @@ -4883,7 +4954,7 @@ void QgsVectorLayer::updateAttributeMapIndex( QgsAttributeMap& map, int oldIndex
map.remove( oldIndex );
}

bool QgsVectorLayer::joinForFieldIndex( int index, QgsVectorJoinInfo& joinInfo, int& indexOffset ) const
const QgsVectorJoinInfo* QgsVectorLayer::joinForFieldIndex( int index, int& indexOffset ) const
{
int currentMaxIndex = 0;
int totIndex = 0;
Expand All @@ -4893,7 +4964,7 @@ bool QgsVectorLayer::joinForFieldIndex( int index, QgsVectorJoinInfo& joinInfo,
{
if( mDataProvider->fields().contains( index ) )
{
return false;
return 0;
}
}
maxIndex( mDataProvider->fields(), totIndex );
Expand All @@ -4911,15 +4982,14 @@ bool QgsVectorLayer::joinForFieldIndex( int index, QgsVectorJoinInfo& joinInfo,

if( joinLayer->pendingFields().contains( index - totIndex ) )
{
joinInfo = *joinIt;
indexOffset = totIndex;
return true;
return &(*joinIt);
}

maxIndex(joinLayer->pendingFields(), currentMaxIndex );
totIndex += (currentMaxIndex + 1);
}

//else an added field
return false;
return 0;
}

0 comments on commit d603091

Please sign in to comment.